M6: gate-m6, operations guide, deviations, screenshots

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP

.github/workflows/ci.yml

@@ -1,6 +1,17 @@
 name: ci
 on: [push, pull_request]
 jobs:
+  lint:
+    # Hard rule (docs/MAINTAINABILITY.md): any finding fails the build.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with: { go-version: stable }
+      - uses: actions/setup-node@v4
+        with: { node-version: 22 }
+      - run: cd web && npm ci
+      - run: make lint
   test:
     strategy:
       matrix:

.gitignore

@@ -1,5 +1,6 @@
 /nmon
 /nmon-arm64
+/nmon-devtools
 /internal/vmsup/bin/*.tar.gz
 /internal/vmsup/bin/victoria-metrics-*
 web/node_modules/
@@ -8,3 +9,5 @@ web/dist/
 *.test
 coverage.out
 /nmon-librenms-import
+/internal/webui/dist/*
+!/internal/webui/dist/.gitkeep

.golangci.yml

@@ -0,0 +1,158 @@
+# Enforces docs/MAINTAINABILITY.md. A hard rule: `make lint` must be clean for any gate to pass.
+version: "2"
+
+run:
+  timeout: 10m
+  tests: true
+
+# Report every finding: funlen and gocognit flag the same line and must both show.
+issues:
+  max-issues-per-linter: 0
+  max-same-issues: 0
+  uniq-by-line: false
+
+linters:
+  default: none
+  enable:
+    - govet
+    - errcheck
+    - staticcheck
+    - unused
+    - ineffassign
+    - funlen
+    - cyclop
+    - gocognit
+    - nestif
+    - dupl
+    - lll
+    - errorlint
+    - usestdlibvars
+    - misspell
+    - revive
+    - depguard
+    - forbidigo
+    - nolintlint
+    - unconvert
+    - bodyclose
+    - copyloopvar
+
+  settings:
+    funlen:
+      lines: 80
+      statements: 50
+      ignore-comments: true
+    cyclop:
+      max-complexity: 15
+    gocognit:
+      min-complexity: 20
+    nestif:
+      min-complexity: 4
+    dupl:
+      threshold: 150
+    lll:
+      line-length: 140
+    misspell:
+      # MikroTik's OS name and a profile name, not a typo of "routers".
+      ignore-rules: [routeros]
+    nolintlint:
+      require-explanation: true
+      require-specific: true
+    revive:
+      rules:
+        - name: exported
+          arguments: ["disableStutteringCheck"]
+        - name: package-comments
+        - name: var-naming
+        - name: indent-error-flow
+        - name: error-strings
+        - name: receiver-naming
+        - name: unexported-return
+    forbidigo:
+      analyze-types: true
+      forbid:
+        - pattern: '^time\.(Now|Sleep|After|Tick|NewTicker|NewTimer)$'
+          msg: "use internal/clock so tests control time (docs/MAINTAINABILITY.md)"
+    depguard:
+      rules:
+        orm-only-in-store:
+          files: ["$all", "!**/internal/store/**"]
+          deny:
+            - pkg: github.com/uptrace/bun
+              desc: SQL and the ORM live only in internal/store
+            - pkg: database/sql
+              desc: SQL lives only in internal/store
+        app-only-from-cmd:
+          files: ["$all", "!**/cmd/**"]
+          deny:
+            - pkg: github.com/v10networks/nmon/internal/app
+              desc: only cmd/* wires the application
+        api-only-from-app:
+          files: ["$all", "!**/internal/app/**", "!**/internal/api/**", "!$test"]
+          deny:
+            - pkg: github.com/v10networks/nmon/internal/api
+              desc: only internal/app mounts the API
+        collection-paths-independent:
+          files:
+            - "**/internal/poller/**"
+            - "**/internal/discovery/**"
+            - "**/internal/traps/**"
+            - "**/internal/ingest/**"
+            - "**/internal/checks/**"
+            - "**/internal/rollup/**"
+            - "**/internal/alert/**"
+          deny:
+            - pkg: github.com/v10networks/nmon/internal/poller
+              desc: collection paths share code via a lower package, not each other
+            - pkg: github.com/v10networks/nmon/internal/discovery
+              desc: collection paths share code via a lower package, not each other
+            - pkg: github.com/v10networks/nmon/internal/traps
+              desc: collection paths share code via a lower package, not each other
+        devtools-not-in-nmon:
+          # The product binary never links dev/test machinery. `nmon profile record` needs only the
+          # .snmprec format, which lives in internal/snmprec.
+          files: ["**/cmd/nmon/**", "!$test"]
+          deny:
+            - pkg: github.com/v10networks/nmon/internal/simagent
+              desc: the simulator belongs to cmd/nmon-devtools and tests, never the shipped binary
+            - pkg: github.com/v10networks/nmon/internal/fixturegen
+              desc: the fixture generator belongs to cmd/nmon-devtools and tests, never the shipped binary
+            - pkg: github.com/v10networks/nmon/internal/fixtures
+              desc: fixtures are test data, never the shipped binary
+            - pkg: github.com/v10networks/nmon/internal/testutil
+              desc: test helpers are never in the shipped binary
+        devtools-no-core:
+          files: ["**/cmd/nmon-devtools/**"]
+          deny:
+            - pkg: github.com/v10networks/nmon/internal/app
+              desc: nmon-devtools talks to a running nmon over HTTP
+            - pkg: github.com/v10networks/nmon/internal/api
+              desc: nmon-devtools talks to a running nmon over HTTP
+        importer-not-in-nmon:
+          files: ["**/cmd/nmon/**"]
+          deny:
+            - pkg: github.com/v10networks/nmon/internal/librenms
+              desc: the LibreNMS importer is a separate tool (DEVIATIONS 9)
+
+  exclusions:
+    paths:
+      - web/node_modules
+    rules:
+      # Table-driven tests are long by nature; every other linter still applies.
+      - path: _test\.go
+        linters: [funlen, cyclop, gocognit, nestif, dupl, lll, forbidigo]
+      # The clock package wraps time; cmd/ and the test-only simulator may use real time.
+      - path: internal/clock/
+        linters: [forbidigo]
+      - path: cmd/
+        linters: [forbidigo]
+      # A package is allowed to import itself's own test helpers and itself.
+      - path: internal/(poller|discovery|traps)/.*_test\.go
+        linters: [depguard]
+
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  exclusions:
+    paths:
+      - web/node_modules

Makefile

@@ -1,24 +1,33 @@
 GO        ?= go
 TAGS      ?= vmembed
-LDFLAGS   := -s -w
+VERSION   ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
+BUILD     ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)-$(shell date -u +%Y%m%d)
+LDFLAGS   := -s -w -X github.com/v10networks/nmon/internal/buildinfo.Version=$(VERSION) -X github.com/v10networks/nmon/internal/buildinfo.Build=$(BUILD)
 BIN       := nmon
 PG_DSN    ?= postgres://nmon:nmon@localhost:55432/nmon?sslmode=disable
 
-.PHONY: all build importer build-arm64 build-noembed fetch-vm web test test-integration test-pg test-v6only vet gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6 clean
+.PHONY: devtools all lint lint-go lint-web build importer build-arm64 build-noembed fetch-vm web test test-integration test-pg test-v6only vet gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6 clean
 
 all: build
 
 fetch-vm:
 	./scripts/fetch-vm.sh
 
-# The UI is built first when web/ exists so embed.FS picks it up.
+# The UI is built first, then copied into internal/webui/dist for go:embed (git-ignored).
 web:
-	@if [ -f web/package.json ]; then cd web && npm ci --no-audit --no-fund && npm run build; fi
+	@if [ -f web/package.json ]; then \
+		cd web && { [ -d node_modules ] || npm ci --no-audit --no-fund; } && npm run build && \
+		cd .. && rm -rf internal/webui/dist/* && cp -r web/dist/. internal/webui/dist/ && touch internal/webui/dist/.gitkeep; \
+	fi
 
-build: fetch-vm
+build: fetch-vm web
 	CGO_ENABLED=0 $(GO) build -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(BIN) ./cmd/nmon
 
-build-arm64: fetch-vm
+# Developer tools (backfill, seed-demo) are a separate binary that is never shipped.
+devtools:
+	CGO_ENABLED=0 $(GO) build -o nmon-devtools ./cmd/nmon-devtools
+
+build-arm64: fetch-vm web
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(GO) build -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(BIN)-arm64 ./cmd/nmon
 
 importer:
@@ -27,6 +36,18 @@ importer:
 build-noembed:
 	CGO_ENABLED=0 $(GO) build -ldflags '$(LDFLAGS)' -o $(BIN) ./cmd/nmon
 
+GOLANGCI := go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.14.0
+
+# Hard rule (docs/MAINTAINABILITY.md): every gate runs this first and it must be clean.
+lint: lint-go lint-web
+
+lint-go:
+	$(GOLANGCI) run ./...
+	./scripts/lint-filesize.sh
+
+lint-web:
+	@if [ -f web/package.json ]; then cd web && npm run lint; fi
+
 vet:
 	$(GO) vet ./...
 
@@ -47,4 +68,4 @@ gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6: gate-m%:
 	./scripts/gate-m$*.sh
 
 clean:
-	rm -f $(BIN) $(BIN)-arm64 nmon-librenms-import
+	rm -f $(BIN) $(BIN)-arm64 nmon-devtools nmon-librenms-import

cmd/nmon-devtools/backfill.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/v10networks/nmon/internal/metrics"
+)
+
+// backfillConfig is what one `devtools backfill` run writes, for every interface in ifaces.
+type backfillConfig struct {
+	hotURL  string
+	device  string
+	ifaces  []string
+	days    int
+	step    time.Duration
+	pattern string
+	meanBps float64 // bits/s; 0 derives the mean from the live rate, else a stable per-interface value
+	seed    uint64
+	hc      *http.Client
+	logf    func(format string, a ...any)
+}
+
+func backfillCmd(ctx context.Context, args []string) error {
+	cfg := backfillConfig{hc: &http.Client{Timeout: 2 * time.Minute}, logf: func(f string, a ...any) { fmt.Printf(f+"\n", a...) }}
+	fs := devFlagSet("backfill")
+	var ifaces stringList
+	var mbps float64
+	fs.StringVar(&cfg.hotURL, "hot-url", "", "base URL of the hot VictoriaMetrics (required; see \"metric stores up\" in the serve log)")
+	fs.StringVar(&cfg.device, "device", "", "device slug (the series' device label)")
+	fs.Var(&ifaces, "iface", "interface ULID; repeat the flag or separate with commas")
+	fs.IntVar(&cfg.days, "days", 31, "days of history to write, ending where the live data begins")
+	fs.DurationVar(&cfg.step, "step", 60*time.Second, "sample spacing")
+	fs.StringVar(&cfg.pattern, "pattern", patternDiurnal, "traffic shape: diurnal or flat")
+	fs.Float64Var(&mbps, "mean-mbps", 0, "mean rate in Mbit/s; 0 follows the live rate when there is one")
+	fs.Uint64Var(&cfg.seed, "seed", 1, "noise seed")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	cfg.ifaces, cfg.meanBps = ifaces, mbps*1e6
+	if err := cfg.validate(); err != nil {
+		return err
+	}
+	for _, id := range cfg.ifaces {
+		if err := cfg.backfillIface(ctx, id); err != nil {
+			return fmt.Errorf("iface %s: %w", id, err)
+		}
+	}
+	return nil
+}
+
+func (c *backfillConfig) validate() error {
+	switch {
+	case c.hotURL == "":
+		return errors.New("--hot-url is required")
+	case c.device == "" || len(c.ifaces) == 0:
+		return errors.New("--device and at least one --iface are required")
+	case c.days < 1 || c.step < time.Second:
+		return errors.New("--days must be at least 1 and --step at least 1s")
+	case c.pattern != patternDiurnal && c.pattern != patternFlat:
+		return fmt.Errorf("unknown --pattern %q (diurnal or flat)", c.pattern)
+	}
+	c.hotURL = strings.TrimRight(c.hotURL, "/")
+	return nil
+}
+
+// liveSeries is what the hot store already holds for one series: the first
+// sample (where history must end and which value it must join) and the live rate.
+type liveSeries struct {
+	ts   time.Time
+	val  float64
+	rate float64 // units per second between the first two samples; 0 when unknown
+	ok   bool
+}
+
+// backfillIface writes in/out octet counters and the oper status for one interface.
+func (c *backfillConfig) backfillIface(ctx context.Context, iface string) error {
+	n := c.days * int(24*time.Hour/c.step)
+	var samples []metrics.Sample
+	var end time.Time
+	for d, dir := range []struct {
+		metric string
+		salt   uint64
+	}{{"ifHCInOctets", 0x1111}, {"ifHCOutOctets", 0x2222}} {
+		live, err := c.probe(ctx, dir.metric, iface)
+		if err != nil {
+			return err
+		}
+		if d == 0 {
+			end = c.endTime(live)
+		}
+		samples = append(samples, c.counter(dir.metric, iface, end, n, live, c.seed^dir.salt)...)
+	}
+	status := 1.0
+	if live, err := c.probe(ctx, "ifOperStatus", iface); err == nil && live.ok {
+		status = live.val
+	}
+	for i := 0; i < n; i++ {
+		ts := end.Add(-time.Duration(n-1-i) * c.step)
+		samples = append(samples, metrics.New("ifOperStatus", status, ts, metrics.LabelDevice, c.device, metrics.LabelIface, iface))
+	}
+	if err := c.send(ctx, samples); err != nil {
+		return err
+	}
+	c.logf("backfilled %s/%s: %d days at %s (%d samples)", c.device, iface, c.days, c.step, len(samples))
+	return nil
+}
+
+// endTime is one step before the first live sample, or one step ago when there is none.
+func (c *backfillConfig) endTime(live liveSeries) time.Time {
+	if live.ok {
+		return live.ts.Add(-c.step).Truncate(time.Millisecond)
+	}
+	return time.Now().Add(-c.step).Truncate(time.Millisecond)
+}
+
+// counter builds one direction's history ending at end. When the live counter
+// is known and large enough, the history is shifted so its last value is where
+// the live counter was one step earlier and the join shows no reset.
+func (c *backfillConfig) counter(metric, iface string, end time.Time, n int, live liveSeries, salt uint64) []metrics.Sample {
+	mean := c.meanBps / 8
+	if mean == 0 && live.rate > 0 {
+		mean = live.rate
+	}
+	if mean == 0 {
+		mean = defaultMeanBytesPerSec(iface, salt)
+	}
+	start := end.Add(-time.Duration(n-1) * c.step)
+	vals := counterSeries(c.pattern, start, n, c.step, mean, salt)
+	var offset float64
+	if live.ok {
+		offset = live.val - mean*c.step.Seconds() - vals[n-1]
+		if offset < 0 {
+			c.logf("warning: %s %s: live counter %.3g is below the %.3g bytes of history; the join will look like a counter reset",
+				c.device, metric, live.val, vals[n-1])
+			offset = 0
+		}
+	}
+	out := make([]metrics.Sample, n)
+	for i, v := range vals {
+		ts := start.Add(time.Duration(i) * c.step)
+		out[i] = metrics.New(metric, math.Floor(v+offset), ts, metrics.LabelDevice, c.device, metrics.LabelIface, iface)
+	}
+	return out
+}
+
+// probe reads the earliest samples of metric{iface=...} from the hot store.
+func (c *backfillConfig) probe(ctx context.Context, metric, iface string) (liveSeries, error) {
+	q := url.Values{"match[]": {fmt.Sprintf(`%s{iface=%q}`, metric, iface)}}
+	var raw bytes.Buffer
+	if err := c.get(ctx, c.hotURL+"/api/v1/export?"+q.Encode(), &raw); err != nil {
+		return liveSeries{}, err
+	}
+	return parseExport(raw.Bytes())
+}

cmd/nmon-devtools/backfilllong.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/v10networks/nmon/internal/longq"
+	"github.com/v10networks/nmon/internal/metrics"
+	"github.com/v10networks/nmon/internal/rollup"
+)
+
+// backfillLongCmd writes rollup series straight into the long store, as if the rollup job had been running
+// for years: hourly average and maximum rates for --months months and 5-minute rates for --days days, flat
+// at the given rates so a gate can assert exact values. The data ends --end-offset ago, leaving the recent
+// hours to the real rollup job (its repair writes higher versions, which win).
+func backfillLongCmd(ctx context.Context, args []string) error {
+	fs := devFlagSet("backfill-long")
+	var ifaces stringList
+	long := fs.String("long-url", "", "base URL of the long VictoriaMetrics (see \"metric stores up\" in the serve log)")
+	device := fs.String("device", "", "device slug (the series' device label)")
+	fs.Var(&ifaces, "iface", "interface ULID; repeat the flag or separate with commas")
+	months := fs.Int("months", 25, "months of hourly rollups to write")
+	days := fs.Int("days", 70, "days of 5-minute rates to write")
+	offset := fs.Duration("end-offset", 3*time.Hour, "stop this long before now")
+	in := fs.Float64("in-bps", 8e6, "constant inbound rate")
+	out := fs.Float64("out-bps", 2e6, "constant outbound rate")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	if *long == "" || *device == "" || len(ifaces) == 0 {
+		return fmt.Errorf("backfill-long needs --long-url, --device and --iface: %w", errUsage)
+	}
+	c := &backfillConfig{hotURL: strings.TrimRight(*long, "/"), hc: &http.Client{Timeout: 5 * time.Minute}}
+	end := time.Now().Add(-*offset).Truncate(time.Hour)
+	for _, id := range ifaces {
+		s := longSamples(*device, id, end, *months, *days, map[string]float64{"in": *in, "out": *out})
+		if err := c.send(ctx, s); err != nil {
+			return errors.Join(fmt.Errorf("iface %s", id), err)
+		}
+		fmt.Printf("backfilled long %s/%s: %d months hourly, %d days 5m (%d samples)\n", *device, id, *months, *days, len(s))
+	}
+	return c.flush(ctx)
+}
+
+// flush makes what was just imported searchable at once: VictoriaMetrics shows new rows only after its
+// buffers flush, and a gate that queries right after the import would otherwise see a partial history.
+func (c *backfillConfig) flush(ctx context.Context) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.hotURL+"/internal/force_flush", http.NoBody)
+	if err != nil {
+		return err
+	}
+	resp, err := c.hc.Do(req)
+	if err != nil {
+		return err
+	}
+	_ = resp.Body.Close() // nothing in the reply matters
+	return nil
+}
+
+// longSamples builds the hourly series from end back months months and the 5-minute series from end back days.
+func longSamples(device, iface string, end time.Time, months, days int, rate map[string]float64) []metrics.Sample {
+	var out []metrics.Sample
+	labels := func(dir string, ws time.Time) []string {
+		return []string{"device", device, "iface", iface, "dir", dir, "v", rollup.VersionLabel(ws, 1)}
+	}
+	for dir, bps := range rate {
+		for h := end; h.After(end.AddDate(0, -months, 0)); h = h.Add(-time.Hour) {
+			out = append(out, metrics.New(longq.MetricRateAvg1h, bps, h, labels(dir, h)...),
+				metrics.New(longq.MetricRateMax1h, bps*1.25, h, labels(dir, h)...))
+		}
+		for t := end; t.After(end.AddDate(0, 0, -days)); t = t.Add(-5 * time.Minute) {
+			out = append(out, metrics.New(longq.MetricRate5m, bps, t, labels(dir, t)...))
+		}
+	}
+	return out
+}

cmd/nmon-devtools/fleet.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/gosnmp/gosnmp"
+
+	"github.com/v10networks/nmon/internal/fixturegen"
+	"github.com/v10networks/nmon/internal/profile"
+	"github.com/v10networks/nmon/internal/snmprec"
+)
+
+// demoDevice is one simulated device of `devtools seed-demo`.
+type demoDevice struct {
+	name, site, role string
+	kind             string // counters, big, router, ups, v1, down: what the gate and screenshots look for
+	rec              *snmprec.Recording
+	community        string
+	version          string // v2c unless set
+	hc               bool   // 64-bit counters, so history can be back-filled
+}
+
+// counterBase is added to every simulated 64-bit octet counter so that the
+// back-filled history (about 3e13 bytes for 31 days at 100 Mbit/s) fits below
+// the live counter and the join shows no reset.
+const counterBase = 100_000_000_000_000
+
+const (
+	oidHCIn  = "1.3.6.1.2.1.31.1.1.1.6."
+	oidHCOut = "1.3.6.1.2.1.31.1.1.1.10."
+	oidUp    = "1.3.6.1.2.1.1.3.0"
+)
+
+// animate returns a copy of rec whose 64-bit octet counters advance at a steady
+// per-port rate from counterBase and whose uptime ticks, so successive polls see
+// traffic. Counters that already advance keep their rate and only gain the base.
+func animate(rec *snmprec.Recording) *snmprec.Recording {
+	recs := append([]snmprec.Record(nil), rec.Records()...)
+	for i := range recs {
+		r := &recs[i]
+		switch {
+		case r.Type == gosnmp.Counter64 && (strings.HasPrefix(r.OID, oidHCIn) || strings.HasPrefix(r.OID, oidHCOut)):
+			recs[i] = animateCounter(*r)
+		case r.OID == oidUp && r.Dyn == nil:
+			if v, ok := r.Value.(uint32); ok {
+				r.Dyn, r.Value = &snmprec.Dynamic{Rate: 100, Initial: uint64(v)}, nil
+			}
+		}
+	}
+	return snmprec.NewRecording(recs)
+}
+
+func animateCounter(r snmprec.Record) snmprec.Record {
+	if r.Dyn != nil {
+		d := *r.Dyn
+		d.Initial += counterBase
+		r.Dyn = &d
+		return r
+	}
+	k, _ := strconv.Atoi(r.OID[strings.LastIndexByte(r.OID, '.')+1:])
+	dir := 1
+	if strings.HasPrefix(r.OID, oidHCOut) {
+		dir = 2
+	}
+	mbps := 1 + (k*(31+7*dir))%90
+	r.Dyn = &snmprec.Dynamic{Rate: float64(mbps) * 125_000, Initial: counterBase + uint64(k)*1_000_000_000}
+	r.Value = nil
+	return r
+}
+
+// profileDevice builds a device that a shipped profile matches, with traffic.
+func profileDevice(set *profile.Set, prof, name, site, role, kind string, ports int) (demoDevice, error) {
+	spec, ok := fixturegen.SpecFor(prof)
+	if !ok {
+		return demoDevice{}, fmt.Errorf("no fixture spec for profile %q", prof)
+	}
+	res, ok := set.Lookup(prof)
+	if !ok {
+		return demoDevice{}, fmt.Errorf("no profile %q", prof)
+	}
+	spec.SysName = name
+	if ports > 0 {
+		spec.Ports = ports
+	}
+	rec, err := fixturegen.FromProfile(res, spec)
+	if err != nil {
+		return demoDevice{}, fmt.Errorf("%s: %w", name, err)
+	}
+	return demoDevice{name: name, site: site, role: role, kind: kind, rec: animate(rec), community: "public", hc: true}, nil
+}
+
+// buildFleet is the small network used for live screenshots: a 48-port switch,
+// routers, a firewall, a UPS, one 32-bit device, the counters fixture and one
+// device whose agent is stopped after seeding, so it reads as down.
+func buildFleet(set *profile.Set, countersRec string) ([]demoDevice, error) {
+	counters, err := snmprec.Load(countersRec)
+	if err != nil {
+		return nil, err
+	}
+	fleet := []demoDevice{{
+		name: "yyz1-edge1", site: "yyz1", role: "edge", kind: "counters", rec: animate(counters), community: "public", hc: true,
+	}}
+	for _, p := range []struct {
+		prof, name, site, role, kind string
+		ports                        int
+	}{
+		{"cisco-ios", "yyz1-acc01", "yyz1", "access", "big", 48},
+		{"cisco-ios", "yyz1-acc02", "yyz1", "access", "router", 24},
+		{"cisco-ios", "ott1-acc02", "ott1", "access", "down", 24}, // its agent is stopped once seeding is done
+		{"juniper-junos", "yyz1-core1", "yyz1", "core", "router", 0},
+		{"arista-eos", "yul1-core1", "yul1", "core", "router", 0},
+		{"mikrotik-routeros", "ott1-gw1", "ott1", "gateway", "router", 0},
+		{"fortigate", "yyz1-fw1", "yyz1", "firewall", "router", 0},
+		{"apc-ups", "yyz1-ups1", "yyz1", "power", "ups", 0},
+	} {
+		d, err := profileDevice(set, p.prof, p.name, p.site, p.role, p.kind, p.ports)
+		if err != nil {
+			return nil, err
+		}
+		fleet = append(fleet, d)
+	}
+	// 32-bit counters on 1 Gbit/s ports: the API flags them unreliable and the UI never graphs them.
+	v1 := fixturegen.Generic(fixturegen.Options{Ports: 12, SysName: "yul1-acc03", Counters: true, NamePattern: "Fa0/%d"})
+	fleet = append(fleet, demoDevice{
+		name: "yul1-acc03", site: "yul1", role: "access", kind: "v1", rec: v1, community: "nmon-v1", version: "v1",
+	})
+	return fleet, nil
+}

cmd/nmon-devtools/gatedump.go

@@ -0,0 +1,163 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+)
+
+// gateDumpCmd prints what the M4 gate asserts on, one line per item, so the shell
+// script can count and match lines instead of parsing JSON.
+//
+//	gate-dump sink --file F          what the webhook sink received, in arrival order
+//	gate-dump alerts --api URL       every alert, newest first
+//	gate-dump devices --api URL      every device's reachability
+func gateDumpCmd(ctx context.Context, args []string) error {
+	if len(args) == 0 {
+		return fmt.Errorf("gate-dump sink|alerts|devices")
+	}
+	fs := devFlagSet("gate-dump " + args[0])
+	file := fs.String("file", "", "sink file (sink)")
+	api := fs.String("api", "", "nmon API base URL (alerts, devices)")
+	mapFile := fs.String("map", "", "slug map written by topology-add; slugs are printed as topology names")
+	if err := fs.Parse(args[1:]); err != nil {
+		return flagErr(err)
+	}
+	names, err := readSlugMap(*mapFile)
+	if err != nil {
+		return err
+	}
+	switch args[0] {
+	case "sink":
+		return dumpSink(*file, names)
+	case "alerts":
+		return dumpAlerts(ctx, *api, names)
+	case "devices":
+		return dumpDevices(ctx, *api, names)
+	}
+	return fmt.Errorf("gate-dump: unknown %q", args[0])
+}
+
+func orDash(s []string) string {
+	if len(s) == 0 {
+		return "-"
+	}
+	return strings.Join(s, ",")
+}
+
+// dumpSink prints "rule state device parents=a,b children=N incident=ID" per request received.
+func dumpSink(file string, names slugNames) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = f.Close() }() // read only
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 1<<20), 1<<20)
+	for sc.Scan() {
+		var l sinkLine
+		var n struct {
+			Rule, State        string
+			IncidentID         string `json:"incident_id"`
+			Device             struct{ Slug string }
+			Parents            []string
+			SuppressedChildren int `json:"suppressed_children"`
+		}
+		if json.Unmarshal(sc.Bytes(), &l) != nil || json.Unmarshal(l.Body, &n) != nil {
+			fmt.Println("unparsable - - parents=- children=0 incident=-")
+			continue
+		}
+		fmt.Printf("%s %s %s parents=%s children=%d incident=%s\n", n.Rule, n.State, names.one(n.Device.Slug), orDash(names.of(n.Parents)),
+			n.SuppressedChildren, n.IncidentID)
+	}
+	return sc.Err()
+}
+
+func getJSON(ctx context.Context, api, path string, out any) error {
+	hc := &http.Client{Timeout: 30 * time.Second}
+	return httpJSON(ctx, hc, http.MethodGet, api+path, nil, out)
+}
+
+// dumpAlerts prints "id rule state device suppressed_by=a,b incident=ID acked=yes|no" per alert.
+func dumpAlerts(ctx context.Context, api string, names slugNames) error {
+	var page struct {
+		Items []struct {
+			ID, Rule, State string
+			Device          struct{ Slug string }
+			SuppressedBy    []string  `json:"suppressed_by"`
+			IncidentID      string    `json:"incident_id"`
+			Ack             *struct{} `json:"ack"`
+		} `json:"items"`
+	}
+	if err := getJSON(ctx, api, "/api/v1/alerts?per_page=1000", &page); err != nil {
+		return err
+	}
+	for _, a := range page.Items {
+		acked := "no"
+		if a.Ack != nil {
+			acked = "yes"
+		}
+		fmt.Printf("%s %s %s %s suppressed_by=%s incident=%s acked=%s\n", a.ID, a.Rule, a.State, names.one(a.Device.Slug),
+			orDash(names.of(a.SuppressedBy)), orDash([]string{a.IncidentID}), acked)
+	}
+	return nil
+}
+
+// dumpDevices prints "slug reachability poll_state warnings=a,b" per device.
+func dumpDevices(ctx context.Context, api string, names slugNames) error {
+	var page struct {
+		Items []struct {
+			Slug, Reachability string
+			PollState          string   `json:"poll_state"`
+			Warnings           []string `json:"warnings"`
+		} `json:"items"`
+	}
+	if err := getJSON(ctx, api, "/api/v1/devices?per_page=1000", &page); err != nil {
+		return err
+	}
+	for _, d := range page.Items {
+		fmt.Printf("%s %s %s warnings=%s\n", url.PathEscape(names.one(d.Slug)), d.Reachability, d.PollState, orDash(d.Warnings))
+	}
+	return nil
+}
+
+// slugNames maps nmon slugs back to the names in the topology file.
+type slugNames map[string]string
+
+func readSlugMap(path string) (slugNames, error) {
+	m := slugNames{}
+	if path == "" {
+		return m, nil
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	for _, l := range strings.Split(string(b), "\n") {
+		if f := strings.Fields(l); len(f) == 2 {
+			m[f[0]] = f[1]
+		}
+	}
+	return m, nil
+}
+
+func (m slugNames) one(slug string) string {
+	if n, ok := m[slug]; ok {
+		return n
+	}
+	return slug
+}
+
+func (m slugNames) of(slugs []string) []string {
+	out := make([]string, len(slugs))
+	for i, s := range slugs {
+		out[i] = m.one(s)
+	}
+	return out
+}

cmd/nmon-devtools/m5_test.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestRebaseInfluxMakesTheNewestLineNowAndKeepsSpacing(t *testing.T) {
+	in := "m,host=a v=1 1000000000000000000\n# c\nm,host=a v=2 1000000060000000000\n"
+	now := time.Unix(2_000_000_000, 0)
+	out, n, err := rebaseInflux(in, now)
+	if err != nil || n != 2 {
+		t.Fatalf("n=%d err=%v", n, err)
+	}
+	want := "m,host=a v=1 1999999940000000000\nm,host=a v=2 2000000000000000000\n"
+	if out != want {
+		t.Fatalf("got %q want %q", out, want)
+	}
+	if _, _, err := rebaseInflux("\n# only comments\n", now); err == nil {
+		t.Fatal("empty input accepted")
+	}
+	if !strings.Contains(out, "host=a") {
+		t.Fatal("tags lost")
+	}
+}
+
+func TestProxmoxMockAuthShape(t *testing.T) {
+	for h, want := range map[string]bool{
+		"PVEAPIToken=root@pam!nmon=s3cret": true, "PVEAPIToken=root@pam!nmon=": false, "Bearer x": false, "": false,
+	} {
+		if validPVEAuth(h, "") != want {
+			t.Errorf("%q: want %v", h, want)
+		}
+	}
+	if validPVEAuth("PVEAPIToken=a@b!c=other", "s3cret") {
+		t.Error("wrong secret accepted")
+	}
+}
+
+func TestLongSamplesCarryVersionedHourlyAndFiveMinuteSeries(t *testing.T) {
+	end := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC)
+	s := longSamples("d", "I", end, 1, 1, map[string]float64{"in": 8e6})
+	var h, f int
+	for _, x := range s {
+		switch x.Name {
+		case "nmon_if_rate_bps_5m":
+			f++
+		default:
+			h++
+		}
+	}
+	if f != 288 || h < 2*28*24 {
+		t.Fatalf("5m=%d hourly=%d", f, h)
+	}
+	if s[0].Labels[3].Name != "v" || s[0].Labels[3].Value != "202610001" {
+		t.Fatalf("labels %+v", s[0].Labels)
+	}
+}

cmd/nmon-devtools/main.go

@@ -0,0 +1,142 @@
+// Command nmon-devtools holds developer helpers for the gates and for screenshots: backfill (synthetic
+// history into the hot store) and seed-demo (a simulated network added to a running nmon). It talks to nmon
+// only through its HTTP API and to the hot store through --hot-url, and is never shipped (docs/dev.md).
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+)
+
+var errUsage = errors.New("usage: nmon-devtools backfill|seed-demo|topology-agents|topology-add|webhook-sink|gate-dump|" +
+	"proxmox-mock|push-replay|backfill-long|oidc-mock|bcrypt [flags]" +
+	" (see docs/dev.md)")
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	if err := run(ctx, os.Args[1:]); err != nil {
+		fmt.Fprintln(os.Stderr, "nmon-devtools:", err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, args []string) error {
+	if len(args) == 0 {
+		return errUsage
+	}
+	switch args[0] {
+	case "backfill":
+		return backfillCmd(ctx, args[1:])
+	case "seed-demo":
+		return seedDemoCmd(ctx, args[1:])
+	case "topology-agents":
+		return topologyAgentsCmd(ctx, args[1:])
+	case "topology-add":
+		return topologyAddCmd(ctx, args[1:])
+	case "webhook-sink":
+		return webhookSinkCmd(ctx, args[1:])
+	case "gate-dump":
+		return gateDumpCmd(ctx, args[1:])
+	case "proxmox-mock":
+		return proxmoxMockCmd(ctx, args[1:])
+	case "push-replay":
+		return pushReplayCmd(ctx, args[1:])
+	case "backfill-long":
+		return backfillLongCmd(ctx, args[1:])
+	case "bcrypt":
+		return bcryptCmd(ctx, args[1:])
+	case "oidc-mock":
+		return oidcMockCmd(ctx, args[1:])
+	default:
+		return fmt.Errorf("unknown command %q: %w", args[0], errUsage)
+	}
+}
+
+// flagErr turns flag.ErrHelp into a clean exit.
+func flagErr(err error) error {
+	if errors.Is(err, flag.ErrHelp) {
+		os.Exit(0)
+	}
+	return err
+}
+
+// stringList is a repeatable string flag.
+type stringList []string
+
+func (s *stringList) String() string { return strings.Join(*s, ",") }
+
+func (s *stringList) Set(v string) error {
+	for _, p := range strings.Split(v, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			*s = append(*s, p)
+		}
+	}
+	return nil
+}
+
+// devFlagSet is a FlagSet that prints its own usage to stderr.
+func devFlagSet(name string) *flag.FlagSet {
+	fs := flag.NewFlagSet("nmon-devtools "+name, flag.ContinueOnError)
+	fs.SetOutput(os.Stderr)
+	return fs
+}
+
+// httpJSON sends an optional JSON body (with the API token in NMON_TOKEN, if set) and decodes a 2xx reply into out (nil: discard).
+func httpJSON(ctx context.Context, hc *http.Client, method, url string, in, out any) error {
+	var body io.Reader
+	if in != nil {
+		b, err := json.Marshal(in)
+		if err != nil {
+			return err
+		}
+		body = strings.NewReader(string(b))
+	}
+	req, err := http.NewRequestWithContext(ctx, method, url, body)
+	if err != nil {
+		return err
+	}
+	if in != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	if tok := os.Getenv("NMON_TOKEN"); tok != "" { // an nmon with authentication on needs an admin API token
+		req.Header.Set("Authorization", "Bearer "+tok)
+	}
+	resp, err := hc.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }() // body read to the end below
+	data, err := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("%s %s: %s: %s", method, url, resp.Status, strings.TrimSpace(string(data[:min(len(data), 300)])))
+	}
+	if out == nil {
+		return nil
+	}
+	return json.Unmarshal(data, out)
+}
+
+func sleepCtx(ctx context.Context, d time.Duration) bool {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+		return false
+	case <-t.C:
+		return true
+	}
+}

cmd/nmon-devtools/oidcmock.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/crypto/bcrypt"
+
+	"github.com/v10networks/nmon/internal/auth/oidctest"
+)
+
+// devTB lets the test-only mock issuer run outside a test binary: it supplies the three methods oidctest uses
+// (Helper, Fatal, Cleanup) and leaves the rest of testing.TB nil.
+type devTB struct {
+	testing.TB
+	cleanups []func()
+}
+
+func (*devTB) Helper() {}
+
+func (*devTB) Fatal(args ...any) { panic(fmt.Sprint(args...)) }
+
+func (d *devTB) Cleanup(f func()) { d.cleanups = append(d.cleanups, f) }
+
+func (d *devTB) close() {
+	for i := len(d.cleanups) - 1; i >= 0; i-- {
+		d.cleanups[i]()
+	}
+}
+
+// oidcMockCmd runs internal/auth/oidctest as a standalone OpenID Connect issuer for the M6 gate and for
+// screenshots. The issuer approves every login at once as the "current user"; a control listener changes who
+// that is: GET /user?sub=ID&name=NAME&groups=a,b on --control. The issuer URL is written to --issuer-file.
+func oidcMockCmd(ctx context.Context, args []string) error {
+	fs := devFlagSet("oidc-mock")
+	control := fs.String("control", "[::1]:0", "listen address of the control endpoint")
+	clientID := fs.String("client-id", "nmon", "OIDC client id the issuer accepts")
+	clientSecret := fs.String("client-secret", "s3cret", "OIDC client secret the issuer accepts")
+	groupsClaim := fs.String("groups-claim", "groups", "name of the groups claim")
+	issuerFile := fs.String("issuer-file", "", "file that receives the issuer URL")
+	portFile := fs.String("port-file", "", "file that receives the control port")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	tb := &devTB{}
+	defer tb.close()
+	srv := oidctest.New(tb)
+	srv.ClientID, srv.ClientSecret, srv.GroupsClaim = *clientID, *clientSecret, *groupsClaim
+	ln, err := net.Listen("tcp", *control)
+	if err != nil {
+		return err
+	}
+	if err := writePort(*portFile, ln); err != nil {
+		return err
+	}
+	if *issuerFile != "" {
+		if err := os.WriteFile(*issuerFile, []byte(srv.Issuer()), 0o600); err != nil {
+			return err
+		}
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /user", func(w http.ResponseWriter, r *http.Request) {
+		q := r.URL.Query()
+		u := oidctest.User{Sub: q.Get("sub"), Name: q.Get("name"), Email: q.Get("email")}
+		if g := q.Get("groups"); g != "" {
+			u.Groups = strings.Split(g, ",")
+		}
+		if u.Sub == "" {
+			http.Error(w, "sub is required", http.StatusBadRequest)
+			return
+		}
+		srv.SetUser(u)
+		w.WriteHeader(http.StatusNoContent)
+	})
+	hs := &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second}
+	go func() { <-ctx.Done(); _ = hs.Close() }()
+	fmt.Printf("oidc-mock issuer %s control %s\n", srv.Issuer(), ln.Addr())
+	if err := hs.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		return err
+	}
+	return nil
+}
+
+// bcryptCmd prints the bcrypt hash of its argument: the content of a break-glass password file.
+func bcryptCmd(_ context.Context, args []string) error {
+	if len(args) != 1 {
+		return errors.New("usage: nmon-devtools bcrypt PASSWORD")
+	}
+	h, err := bcrypt.GenerateFromPassword([]byte(args[0]), bcrypt.DefaultCost)
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(h))
+	return nil
+}

cmd/nmon-devtools/pattern.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"math"
+	"time"
+)
+
+// Synthetic traffic shapes for `nmon devtools backfill`. Everything here is a
+// pure function of its inputs, so a back-fill is reproducible and testable.
+
+const (
+	patternDiurnal = "diurnal"
+	patternFlat    = "flat"
+)
+
+// splitmix64 is a tiny deterministic hash used for noise; math/rand would make
+// two runs differ and a global source would be shared state.
+func splitmix64(x uint64) uint64 {
+	x += 0x9e3779b97f4a7c15
+	x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9
+	x = (x ^ (x >> 27)) * 0x94d049bb133111eb
+	return x ^ (x >> 31)
+}
+
+// unit maps a hash to [0, 1).
+func unit(h uint64) float64 { return float64(h>>11) / (1 << 53) }
+
+// rateFactor is the multiple of the mean rate at t: 1 for "flat"; for
+// "diurnal" an evening peak (about 20:00 UTC), an early-morning trough, quieter
+// weekends, per-sample noise and the odd short burst.
+func rateFactor(pattern string, t time.Time, salt uint64) float64 {
+	if pattern == patternFlat {
+		return 1
+	}
+	h := float64(t.Hour()) + float64(t.Minute())/60
+	f := 1 + 0.6*math.Cos(2*math.Pi*(h-20)/24)
+	if wd := t.Weekday(); wd == time.Saturday || wd == time.Sunday {
+		f *= 0.8
+	}
+	slot := uint64(t.Unix() / 60)
+	f *= 0.88 + 0.24*unit(splitmix64(salt^slot))
+	if unit(splitmix64(salt^(slot/4)^0xb5ad4eceda1ce2a9)) < 0.01 { // ~1% of 4-minute slots burst
+		f *= 2.5
+	}
+	return math.Max(f, 0.05)
+}
+
+// counterSeries returns n cumulative octet-counter values, one per step from
+// start, beginning at 0 and never decreasing. mean is bytes per second.
+func counterSeries(pattern string, start time.Time, n int, step time.Duration, mean float64, salt uint64) []float64 {
+	out := make([]float64, n)
+	var total float64
+	for i := 1; i < n; i++ {
+		t := start.Add(time.Duration(i) * step)
+		total += mean * rateFactor(pattern, t, salt) * step.Seconds()
+		out[i] = math.Floor(total)
+	}
+	return out
+}
+
+// defaultMeanBytesPerSec is a stable per-interface mean (2 to 60 Mbit/s) for when
+// the live rate is unknown.
+func defaultMeanBytesPerSec(iface string, salt uint64) float64 {
+	h := salt
+	for _, c := range []byte(iface) {
+		h = splitmix64(h ^ uint64(c))
+	}
+	return (2 + 58*unit(h)*unit(h>>7)) * 125_000
+}

cmd/nmon-devtools/pattern_test.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCounterSeriesIsMonotonicAndDeterministic(t *testing.T) {
+	start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
+	a := counterSeries(patternDiurnal, start, 3*24*60, time.Minute, 1_000_000, 7)
+	b := counterSeries(patternDiurnal, start, 3*24*60, time.Minute, 1_000_000, 7)
+	for i := range a {
+		if a[i] != b[i] {
+			t.Fatalf("not deterministic at %d", i)
+		}
+		if i > 0 && a[i] < a[i-1] {
+			t.Fatalf("decreased at %d", i)
+		}
+	}
+	if a[0] != 0 {
+		t.Fatalf("starts at %v, want 0", a[0])
+	}
+	meanRate := a[len(a)-1] / (float64(len(a)-1) * 60)
+	if meanRate < 0.8e6 || meanRate > 1.4e6 {
+		t.Fatalf("mean rate %.0f B/s drifted from 1e6", meanRate)
+	}
+}
+
+func TestDiurnalEveningBusierThanNight(t *testing.T) {
+	// Monday, so the weekend factor does not interfere. Averaged over many days the noise washes out.
+	var night, evening float64
+	for d := 0; d < 28; d++ {
+		day := time.Date(2026, 9, 7, 0, 0, 0, 0, time.UTC).AddDate(0, 0, d)
+		if wd := day.Weekday(); wd == time.Saturday || wd == time.Sunday {
+			continue
+		}
+		night += rateFactor(patternDiurnal, day.Add(5*time.Hour), 1)
+		evening += rateFactor(patternDiurnal, day.Add(20*time.Hour), 1)
+	}
+	if evening < 2*night {
+		t.Fatalf("evening %.2f is not clearly busier than night %.2f", evening, night)
+	}
+}
+
+func TestFlatPatternIsConstant(t *testing.T) {
+	if f := rateFactor(patternFlat, time.Now(), 3); f != 1 {
+		t.Fatalf("flat factor %v", f)
+	}
+}
+
+func TestDefaultMeanIsStableAndBounded(t *testing.T) {
+	m := defaultMeanBytesPerSec("01abc", 1)
+	if m != defaultMeanBytesPerSec("01abc", 1) || m < 2*125_000 || m > 60*125_000 {
+		t.Fatalf("mean %v", m)
+	}
+}

cmd/nmon-devtools/proxmoxmock.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+// proxmoxMockCmd serves a recorded cluster/resources response the way a Proxmox VE API does, for the M5 gate:
+// GET /api2/json/cluster/resources with "Authorization: PVEAPIToken=<id>=<secret>". Any other path is a 404 and
+// a request without that header shape is a 401, so a client that forgot the secret fails like the real thing.
+func proxmoxMockCmd(ctx context.Context, args []string) error {
+	fs := devFlagSet("proxmox-mock")
+	listen := fs.String("listen", "[::1]:0", "listen address")
+	file := fs.String("file", "", "recorded cluster/resources JSON (testdata/proxmox/cluster-resources.json)")
+	secret := fs.String("secret", "", "token secret to require (empty: any non-empty secret)")
+	portFile := fs.String("port-file", "", "file that receives the bound port")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	body, err := os.ReadFile(*file)
+	if err != nil || *file == "" {
+		return fmt.Errorf("proxmox-mock needs --file: %w", errors.Join(err, errUsage))
+	}
+	ln, err := net.Listen("tcp", *listen)
+	if err != nil {
+		return err
+	}
+	if err := writePort(*portFile, ln); err != nil {
+		return err
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("GET /api2/json/cluster/resources", func(w http.ResponseWriter, r *http.Request) {
+		if !validPVEAuth(r.Header.Get("Authorization"), *secret) {
+			http.Error(w, `{"data":null}`, http.StatusUnauthorized)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write(body) // a client that hung up has nobody to tell
+	})
+	srv := &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second}
+	go func() { <-ctx.Done(); _ = srv.Close() }()
+	fmt.Printf("proxmox-mock listening on %s\n", ln.Addr())
+	if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		return err
+	}
+	return nil
+}
+
+// validPVEAuth accepts "PVEAPIToken=user@realm!id=secret" with the wanted secret (or any, when want is empty).
+func validPVEAuth(h, want string) bool {
+	rest, ok := strings.CutPrefix(h, "PVEAPIToken=")
+	if !ok {
+		return false
+	}
+	_, got, ok := strings.Cut(rest, "=")
+	return ok && got != "" && (want == "" || got == want)
+}
+
+// writePort records the bound port for a shell script to read.
+func writePort(file string, ln net.Listener) error {
+	if file == "" {
+		return nil
+	}
+	tcp, ok := ln.Addr().(*net.TCPAddr)
+	if !ok {
+		return fmt.Errorf("listener is not TCP: %s", ln.Addr())
+	}
+	return os.WriteFile(file, []byte(fmt.Sprint(tcp.Port)), 0o600)
+}

cmd/nmon-devtools/pushreplay.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// pushReplayCmd sends a recorded Influx line-protocol file to a running nmon's POST /write, shifting every
+// timestamp so the newest line is "now": a recording is days old by the time a gate replays it, and ingest
+// refuses timestamps far from the clock.
+func pushReplayCmd(ctx context.Context, args []string) error {
+	fs := devFlagSet("push-replay")
+	url := fs.String("url", "", "ingest URL, for example http://127.0.0.1:8443/write")
+	token := fs.String("token", "", "push token secret (omit to test the unauthenticated case)")
+	file := fs.String("file", "", "Influx line-protocol file")
+	extra := fs.String("append", "", "one more line to send after the file (a raw line, rebased like the others)")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	raw, err := os.ReadFile(*file)
+	if err != nil || *url == "" {
+		return fmt.Errorf("push-replay needs --url and --file: %w", errors.Join(err, errUsage))
+	}
+	body, n, err := rebaseInflux(string(raw)+"\n"+*extra, time.Now())
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, *url, bytes.NewReader([]byte(body)))
+	if err != nil {
+		return err
+	}
+	if *token != "" {
+		req.Header.Set("Authorization", "Bearer "+*token)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }() // body read below
+	msg, _ := io.ReadAll(io.LimitReader(resp.Body, 500))
+	fmt.Printf("%d lines -> HTTP %d %s\n", n, resp.StatusCode, strings.TrimSpace(string(msg)))
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("push refused: HTTP %d", resp.StatusCode)
+	}
+	return nil
+}
+
+// rebaseInflux shifts the nanosecond timestamp at the end of every line by the same amount so the newest one
+// equals now. Lines without a timestamp, blank lines and comments pass through.
+func rebaseInflux(text string, now time.Time) (string, int, error) {
+	lines := strings.Split(text, "\n")
+	var newest int64
+	for _, l := range lines {
+		if ts, ok := lineTS(l); ok {
+			newest = max(newest, ts)
+		}
+	}
+	shift := now.UnixNano() - newest
+	var out strings.Builder
+	n := 0
+	for _, l := range lines {
+		l = strings.TrimSpace(l)
+		if l == "" || strings.HasPrefix(l, "#") {
+			continue
+		}
+		n++
+		if ts, ok := lineTS(l); ok {
+			l = l[:strings.LastIndexByte(l, ' ')+1] + strconv.FormatInt(ts+shift, 10)
+		}
+		out.WriteString(l + "\n")
+	}
+	if n == 0 {
+		return "", 0, errors.New("no lines to send")
+	}
+	return out.String(), n, nil
+}
+
+// lineTS returns the trailing nanosecond timestamp of an Influx line, if it has one.
+func lineTS(l string) (int64, bool) {
+	l = strings.TrimSpace(l)
+	i := strings.LastIndexByte(l, ' ')
+	if l == "" || strings.HasPrefix(l, "#") || i < 0 {
+		return 0, false
+	}
+	ts, err := strconv.ParseInt(l[i+1:], 10, 64)
+	return ts, err == nil && ts > 1e15
+}

cmd/nmon-devtools/seed.go

@@ -0,0 +1,242 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/v10networks/nmon/internal/profile"
+	"github.com/v10networks/nmon/internal/simagent"
+)
+
+// seedConfig is the command line of `devtools seed-demo`.
+type seedConfig struct {
+	api, hotURL string
+	days        int
+	step        time.Duration
+	countersRec string
+	envOut      string
+	profilesDir string
+	workers     int
+	hc          *http.Client
+}
+
+// seeded is one device after it was added to nmon.
+type seeded struct {
+	demoDevice
+	slug   string
+	agent  *simagent.Agent
+	ip     string
+	ifaces []seedIface
+}
+
+type seedIface struct {
+	ID         string `json:"id"`
+	Name       string `json:"ifname"`
+	HC         bool   `json:"has_hc_counters"`
+	Excluded   bool   `json:"excluded"`
+	Unreliable bool   `json:"unreliable"`
+}
+
+func (i seedIface) graphable() bool { return i.HC && !i.Excluded && !i.Unreliable }
+
+func seedDemoCmd(ctx context.Context, args []string) error {
+	cfg := seedConfig{hc: &http.Client{Timeout: 3 * time.Minute}}
+	fs := devFlagSet("seed-demo")
+	fs.StringVar(&cfg.api, "api", "http://127.0.0.1:8443", "base URL of the running nmon (started with --insecure-http)")
+	fs.StringVar(&cfg.hotURL, "hot-url", "", "base URL of the hot VictoriaMetrics, for the back-fill (required)")
+	fs.IntVar(&cfg.days, "days", 31, "days of history to back-fill")
+	fs.DurationVar(&cfg.step, "step", 60*time.Second, "back-fill sample spacing")
+	fs.StringVar(&cfg.countersRec, "counters-rec", "testdata/snmprec/counters.snmprec", "the counters fixture recording")
+	fs.StringVar(&cfg.envOut, "env-out", "", "write NMON_DEVICE=<counters slug> and NMON_DEVICE_48=<48-port slug> to this file when ready")
+	fs.StringVar(&cfg.profilesDir, "profiles-dir", "", "operator profiles directory (shipped profiles are always loaded)")
+	fs.IntVar(&cfg.workers, "workers", 6, "parallel back-fill workers")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	if cfg.hotURL == "" {
+		return errors.New("--hot-url is required")
+	}
+	cfg.api, cfg.hotURL = strings.TrimRight(cfg.api, "/"), strings.TrimRight(cfg.hotURL, "/")
+	return cfg.run(ctx)
+}
+
+func (c *seedConfig) run(ctx context.Context) error {
+	set, err := profile.Load(c.profilesDir)
+	if err != nil {
+		return err
+	}
+	fleet, err := buildFleet(set, c.countersRec)
+	if err != nil {
+		return err
+	}
+	var pool simagent.Pool
+	defer func() { _ = pool.Close() }() // process is exiting
+	added, err := c.addAll(ctx, &pool, fleet)
+	if err != nil {
+		return err
+	}
+	if err := c.backfillAll(ctx, added); err != nil {
+		return err
+	}
+	if err := c.writeEnv(added); err != nil {
+		return err
+	}
+	fmt.Println("seed-demo ready: agents keep answering until interrupted")
+	<-ctx.Done()
+	return nil
+}
+
+// addAll starts one agent per device on [::1] with its own port and adds the
+// device through the API with that port.
+func (c *seedConfig) addAll(ctx context.Context, pool *simagent.Pool, fleet []demoDevice) ([]seeded, error) {
+	var out []seeded
+	for i, d := range fleet {
+		cfg := simagent.Config{Addr: "[::1]:0", Recording: d.rec, Community: d.community}
+		if err := pool.Start(ctx, []simagent.Config{cfg}); err != nil {
+			return nil, fmt.Errorf("%s: %w", d.name, err)
+		}
+		agent := pool.Agents()[i]
+		s, err := c.addDevice(ctx, d, "::1", int(agent.Addr().Port()))
+		s.agent = agent
+		if err != nil {
+			return nil, fmt.Errorf("add %s: %w", d.name, err)
+		}
+		fmt.Printf("added %-12s %-10s %s (%d interfaces)\n", d.name, d.kind, s.slug, len(s.ifaces))
+		out = append(out, s)
+	}
+	return out, nil
+}
+
+func (c *seedConfig) addDevice(ctx context.Context, d demoDevice, ip string, port int) (seeded, error) {
+	version := d.version
+	if version == "" {
+		version = "v2c"
+	}
+	body := map[string]any{
+		"name": d.name, "addresses": []string{ip}, "site": d.site, "role": d.role,
+		"snmp": map[string]any{"version": version, "community": d.community, "port": port},
+	}
+	var created struct {
+		Slug string `json:"slug"`
+	}
+	if err := httpJSON(ctx, c.hc, http.MethodPost, c.api+"/api/v1/devices", body, &created); err != nil {
+		return seeded{}, err
+	}
+	s := seeded{demoDevice: d, slug: created.Slug, ip: ip}
+	var got struct {
+		Interfaces []seedIface `json:"interfaces"`
+	}
+	if err := httpJSON(ctx, c.hc, http.MethodGet, c.api+"/api/v1/devices/"+url.PathEscape(created.Slug), nil, &got); err != nil {
+		return seeded{}, err
+	}
+	if len(got.Interfaces) == 0 {
+		return seeded{}, errors.New("discovery found no interfaces")
+	}
+	s.ifaces = got.Interfaces
+	return s, nil
+}
+
+// backfillAll waits for each device's first live samples, then writes history
+// for every graphable interface in parallel.
+func (c *seedConfig) backfillAll(ctx context.Context, devs []seeded) error {
+	type job struct{ dev, iface string }
+	var jobs []job
+	probe := &backfillConfig{hotURL: c.hotURL, hc: c.hc}
+	for _, d := range devs {
+		for _, i := range d.ifaces {
+			if d.hc && i.graphable() {
+				jobs = append(jobs, job{d.slug, i.ID})
+			}
+		}
+		if err := c.waitLive(ctx, probe, d); err != nil {
+			return err
+		}
+		if d.kind == "down" {
+			_ = d.agent.Close() // goes silent now, so it reads as down by the time the screenshots are taken
+		}
+	}
+	ch := make(chan job)
+	errs := make(chan error, len(jobs))
+	var wg sync.WaitGroup
+	for w := 0; w < max(1, c.workers); w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := range ch {
+				cfg := &backfillConfig{hotURL: c.hotURL, device: j.dev, days: c.days, step: c.step, pattern: patternDiurnal,
+					seed: 1, hc: c.hc, logf: func(string, ...any) {}}
+				errs <- cfg.backfillIface(ctx, j.iface)
+			}
+		}()
+	}
+	for _, j := range jobs {
+		ch <- j
+	}
+	close(ch)
+	wg.Wait()
+	close(errs)
+	for err := range errs {
+		if err != nil {
+			return err
+		}
+	}
+	fmt.Printf("back-filled %d interfaces, %d days at %s\n", len(jobs), c.days, c.step)
+	return nil
+}
+
+// waitLive blocks until the hot store holds two live samples of the device's
+// first graphable interface, so the back-fill can end where they begin.
+func (c *seedConfig) waitLive(ctx context.Context, probe *backfillConfig, d seeded) error {
+	if !d.hc {
+		return nil
+	}
+	var first string
+	for _, i := range d.ifaces {
+		if i.graphable() {
+			first = i.ID
+			break
+		}
+	}
+	if first == "" {
+		return nil
+	}
+	deadline := time.Now().Add(3 * time.Minute)
+	for time.Now().Before(deadline) {
+		live, err := probe.probe(ctx, "ifHCInOctets", first)
+		if err == nil && live.ok && live.rate > 0 {
+			return nil
+		}
+		if !sleepCtx(ctx, 2*time.Second) {
+			return ctx.Err()
+		}
+	}
+	return fmt.Errorf("%s: no live samples in the hot store after 3 minutes (is nmon polling? --poll-interval)", d.slug)
+}
+
+func (c *seedConfig) writeEnv(devs []seeded) error {
+	var b bytes.Buffer
+	for _, d := range devs {
+		switch d.kind {
+		case "counters":
+			fmt.Fprintf(&b, "NMON_DEVICE=%s\n", d.slug)
+		case "big":
+			fmt.Fprintf(&b, "NMON_DEVICE_48=%s\n", d.slug)
+		case "v1":
+			fmt.Fprintf(&b, "NMON_DEVICE_32BIT=%s\n", d.slug)
+		case "down":
+			fmt.Fprintf(&b, "NMON_DEVICE_DOWN=%s\n", d.slug)
+		}
+	}
+	if c.envOut == "" {
+		return nil
+	}
+	return os.WriteFile(c.envOut, b.Bytes(), 0o600)
+}

cmd/nmon-devtools/store.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/v10networks/nmon/internal/metrics"
+)
+
+// parseExport reads VictoriaMetrics' /api/v1/export (JSON lines) and returns
+// the earliest sample of the first series.
+func parseExport(raw []byte) (liveSeries, error) {
+	line, _, _ := bytes.Cut(bytes.TrimSpace(raw), []byte("\n"))
+	if len(line) == 0 {
+		return liveSeries{}, nil
+	}
+	var s struct {
+		Values     []float64 `json:"values"`
+		Timestamps []int64   `json:"timestamps"`
+	}
+	if err := json.Unmarshal(line, &s); err != nil {
+		return liveSeries{}, fmt.Errorf("decode export: %w", err)
+	}
+	if len(s.Values) == 0 || len(s.Values) != len(s.Timestamps) {
+		return liveSeries{}, nil
+	}
+	out := liveSeries{ts: time.UnixMilli(s.Timestamps[0]).UTC(), val: s.Values[0], ok: true}
+	if len(s.Values) > 1 && s.Timestamps[1] > s.Timestamps[0] {
+		out.rate = (s.Values[1] - s.Values[0]) / (float64(s.Timestamps[1]-s.Timestamps[0]) / 1000)
+	}
+	return out, nil
+}
+
+func (c *backfillConfig) get(ctx context.Context, u string, dst io.Writer) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := c.hc.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }() // read to the end below
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
+		return fmt.Errorf("GET %s: %s: %s", u, resp.Status, b)
+	}
+	_, err = io.Copy(dst, resp.Body)
+	return err
+}
+
+// importBatch is how many samples go in one POST (about 5 MB of text).
+const importBatch = 50_000
+
+// send writes samples through /api/v1/import/prometheus in batches.
+func (c *backfillConfig) send(ctx context.Context, samples []metrics.Sample) error {
+	for len(samples) > 0 {
+		n := min(len(samples), importBatch)
+		body := make([]byte, 0, n*110)
+		for _, s := range samples[:n] {
+			body = metrics.AppendPromText(body, s)
+		}
+		req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.hotURL+"/api/v1/import/prometheus", bytes.NewReader(body))
+		if err != nil {
+			return err
+		}
+		resp, err := c.hc.Do(req)
+		if err != nil {
+			return err
+		}
+		msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
+		_ = resp.Body.Close() // reply already read
+		if resp.StatusCode/100 != 2 {
+			return fmt.Errorf("import: %s: %s", resp.Status, msg)
+		}
+		samples = samples[n:]
+	}
+	return nil
+}

cmd/nmon-devtools/topology.go

@@ -0,0 +1,219 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/v10networks/nmon/internal/deps"
+	"github.com/v10networks/nmon/internal/fixturegen"
+	"github.com/v10networks/nmon/internal/simagent"
+)
+
+// Topology commands for the M4 gate: one simulated SNMP agent per device of a
+// topology file (deps.Topology), the devices added to a running nmon with their
+// parents, and a webhook receiver that records what nmon sends.
+
+const agentCommunity = "public"
+
+// topologyPort is the UDP port of the i-th device of a topology.
+func topologyPort(base, i int) int { return base + i }
+
+// topologyAgentsCmd runs one agent per device on [::1]:port-base+index. The
+// agent of device X is down while the file <control-dir>/stop-X exists, so a
+// shell script can take a device away and bring it back on the same port.
+func topologyAgentsCmd(ctx context.Context, args []string) error {
+	fs := devFlagSet("topology-agents")
+	file := fs.String("topology", "", "topology file (testdata/topologies/*.yaml)")
+	base := fs.Int("port-base", 0, "UDP port of the first device; device i uses port-base+i")
+	ctl := fs.String("control-dir", "", "directory watched for stop-<slug> files")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	if *file == "" || *base == 0 || *ctl == "" {
+		return fmt.Errorf("topology-agents needs --topology, --port-base and --control-dir")
+	}
+	topo, err := deps.LoadTopology(*file)
+	if err != nil {
+		return err
+	}
+	f := &agentFleet{topo: topo, base: *base, ctl: *ctl, agents: map[string]*simagent.Agent{}}
+	defer f.closeAll()
+	for {
+		if err := f.reconcile(ctx); err != nil {
+			return err
+		}
+		if !f.announced {
+			f.announced = true
+			fmt.Printf("agents ready: %d devices on [::1]:%d-%d\n", len(topo.Devices), *base, topologyPort(*base, len(topo.Devices)-1))
+		}
+		if !sleepCtx(ctx, 200*time.Millisecond) {
+			return nil
+		}
+	}
+}
+
+type agentFleet struct {
+	topo      *deps.Topology
+	base      int
+	ctl       string
+	agents    map[string]*simagent.Agent
+	announced bool
+}
+
+// reconcile starts the agents whose stop file is gone and stops those whose stop file appeared.
+func (f *agentFleet) reconcile(ctx context.Context) error {
+	for i, d := range f.topo.Devices {
+		_, stopped := os.Stat(filepath.Join(f.ctl, "stop-"+d.Slug))
+		want := stopped != nil // no stop file: the agent should run
+		a := f.agents[d.Slug]
+		switch {
+		case want && a == nil:
+			rec := fixturegen.Generic(fixturegen.Options{Ports: 4, HCCounters: true, Counters: true, SysName: d.Slug})
+			cfg := simagent.Config{Addr: fmt.Sprintf("[::1]:%d", topologyPort(f.base, i)), Recording: rec, Community: agentCommunity}
+			ag, err := simagent.Start(ctx, cfg)
+			if err != nil {
+				return fmt.Errorf("agent %s: %w", d.Slug, err)
+			}
+			f.agents[d.Slug] = ag
+		case !want && a != nil:
+			_ = a.Close() // best-effort: the port is released either way
+			delete(f.agents, d.Slug)
+		}
+	}
+	return nil
+}
+
+func (f *agentFleet) closeAll() {
+	for _, a := range f.agents {
+		_ = a.Close() // best-effort at exit
+	}
+}
+
+// topologyAddCmd adds every device of a topology to a running nmon, pointing at
+// its agent, then sets the parents by slug.
+func topologyAddCmd(ctx context.Context, args []string) error {
+	fs := devFlagSet("topology-add")
+	api := fs.String("api", "", "nmon API base URL")
+	file := fs.String("topology", "", "topology file")
+	base := fs.Int("port-base", 0, "UDP port of the first device's agent")
+	mapOut := fs.String("map-out", "", "file that receives one \"<nmon slug> <topology name>\" line per device")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	if *api == "" || *file == "" || *base == 0 {
+		return fmt.Errorf("topology-add needs --api, --topology and --port-base")
+	}
+	topo, err := deps.LoadTopology(*file)
+	if err != nil {
+		return err
+	}
+	hc := &http.Client{Timeout: 90 * time.Second}
+	slugOf := map[string]string{} // topology name -> the slug nmon generated
+	for i, d := range topo.Devices {
+		body := map[string]any{
+			"name": d.Slug, "addresses": []string{"::1"}, "role": d.Role, "kind": d.Kind,
+			"snmp": map[string]any{"version": "v2c", "community": agentCommunity, "port": topologyPort(*base, i)},
+		}
+		var got struct {
+			Slug string `json:"slug"`
+		}
+		if err := httpJSON(ctx, hc, http.MethodPost, *api+"/api/v1/devices", body, &got); err != nil {
+			return fmt.Errorf("add %s: %w", d.Slug, err)
+		}
+		slugOf[d.Slug] = got.Slug // nmon appends a random suffix to the slug it derives from the name
+	}
+	for _, d := range topo.Devices {
+		if len(d.Parents) == 0 {
+			continue
+		}
+		parents := make([]string, len(d.Parents))
+		for i, p := range d.Parents {
+			parents[i] = slugOf[p]
+		}
+		err := httpJSON(ctx, hc, http.MethodPut, *api+"/api/v1/devices/"+url.PathEscape(slugOf[d.Slug]), map[string]any{"parents": parents}, nil)
+		if err != nil {
+			return fmt.Errorf("parents of %s: %w", d.Slug, err)
+		}
+	}
+	fmt.Printf("topology-add ready: %d devices\n", len(topo.Devices))
+	return writeSlugMap(*mapOut, topo, slugOf)
+}
+
+// writeSlugMap records which nmon slug belongs to which topology name, for gate-dump --map and the gate script.
+func writeSlugMap(path string, topo *deps.Topology, slugOf map[string]string) error {
+	if path == "" {
+		return nil
+	}
+	var b strings.Builder
+	for _, d := range topo.Devices {
+		fmt.Fprintf(&b, "%s %s\n", slugOf[d.Slug], d.Slug)
+	}
+	return os.WriteFile(path, []byte(b.String()), 0o600)
+}
+
+// sinkLine is one recorded request.
+type sinkLine struct {
+	At   string          `json:"at"`
+	Path string          `json:"path"`
+	Body json.RawMessage `json:"body"`
+}
+
+// webhookSinkCmd records every POST body as one JSON line in --out, answers 200,
+// and writes the bound port to --port-file once it listens.
+func webhookSinkCmd(ctx context.Context, args []string) error {
+	fs := devFlagSet("webhook-sink")
+	listen := fs.String("listen", "[::1]:0", "listen address")
+	out := fs.String("out", "", "file that receives one JSON line per request")
+	portFile := fs.String("port-file", "", "file that receives the bound port")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	if *out == "" {
+		return fmt.Errorf("webhook-sink needs --out")
+	}
+	f, err := os.OpenFile(*out, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = f.Close() }() // lines are written whole; nothing is buffered
+	ln, err := net.Listen("tcp", *listen)
+	if err != nil {
+		return err
+	}
+	if *portFile != "" {
+		port := ln.Addr().(*net.TCPAddr).Port
+		if err := os.WriteFile(*portFile, []byte(fmt.Sprint(port)), 0o600); err != nil {
+			return err
+		}
+	}
+	var mu sync.Mutex
+	srv := &http.Server{ReadHeaderTimeout: 10 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, _ := io.ReadAll(io.LimitReader(r.Body, 1<<20))
+		raw := json.RawMessage(b)
+		if !json.Valid(b) {
+			raw, _ = json.Marshal(string(b))
+		}
+		line, _ := json.Marshal(sinkLine{At: time.Now().UTC().Format(time.RFC3339Nano), Path: r.URL.Path, Body: raw})
+		mu.Lock()
+		_, _ = f.Write(append(line, '\n'))
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	})}
+	go func() { <-ctx.Done(); _ = srv.Close() }()
+	fmt.Printf("webhook-sink listening on %s\n", ln.Addr())
+	if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		return err
+	}
+	return nil
+}

cmd/nmon-librenms-import/main.go

@@ -0,0 +1,110 @@
+// Command nmon-librenms-import turns LibreNMS OS definitions into draft nmon
+// profiles. It runs offline, before nmon; nmon itself neither links nor exposes it.
+//
+//	nmon-librenms-import --librenms <checkout> --out <dir> [--os junos,ios,...] [--mibs-dir <dir>]...
+//
+// The drafts hold only type: unverified metrics. Review them, copy the ones you
+// want into <data-dir>/profiles/ and confirm them with `nmon profile test`.
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/v10networks/nmon/internal/cliout"
+	"github.com/v10networks/nmon/internal/librenms"
+)
+
+type listFlag []string
+
+func (l *listFlag) String() string { return strings.Join(*l, ",") }
+func (l *listFlag) Set(v string) error {
+	for _, s := range strings.Split(v, ",") {
+		if s = strings.TrimSpace(s); s != "" {
+			*l = append(*l, s)
+		}
+	}
+	return nil
+}
+
+func main() {
+	if err := run(os.Args[1:]); err != nil {
+		fmt.Fprintln(os.Stderr, "nmon-librenms-import:", err)
+		os.Exit(1)
+	}
+}
+
+func run(args []string) error {
+	fs := flag.NewFlagSet("nmon-librenms-import", flag.ContinueOnError)
+	var (
+		root, out string
+		oses      listFlag
+		mibDirs   listFlag
+	)
+	fs.StringVar(&root, "librenms", "", "path of a LibreNMS checkout (required)")
+	fs.StringVar(&out, "out", "", "directory to write <os>.yaml drafts into (required)")
+	fs.Var(&oses, "os", "comma-separated LibreNMS OS names, e.g. junos,ios (default: every OS)")
+	fs.Var(&mibDirs, "mibs-dir", "extra MIB directory searched before the checkout's mibs/; repeatable")
+	fs.Usage = func() {
+		// Usage text is best-effort; a failed write to the terminal has nowhere to go.
+		_, _ = fmt.Fprintln(fs.Output(), "usage: nmon-librenms-import --librenms DIR --out DIR [--os a,b] [--mibs-dir DIR]...")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return nil
+		}
+		return err
+	}
+	if root == "" || out == "" || fs.NArg() > 0 {
+		fs.Usage()
+		return errors.New("--librenms and --out are required")
+	}
+	imp, err := librenms.New(librenms.Options{LibreNMS: root, MIBDirs: mibDirs})
+	if err != nil {
+		return err
+	}
+	if len(oses) == 0 {
+		if oses, err = imp.OSList(); err != nil {
+			return err
+		}
+	}
+	if err := os.MkdirAll(out, 0o755); err != nil {
+		return err
+	}
+	return importAll(imp, oses, out, os.Stderr)
+}
+
+// importAll imports each OS, writes its draft into out and prints a summary
+// table to w. Output failures count as failures of the run.
+func importAll(imp *librenms.Importer, oses []string, out string, w io.Writer) error {
+	tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
+	cw := cliout.New(tw)
+	cw.Println("OS\tPROFILE\tMAPPED\tLABELS\tSKIPPED")
+	failed := 0
+	for _, name := range oses {
+		d, err := imp.Import(name)
+		if err != nil {
+			cw.Printf("%s\tERROR\t\t\t%v\n", name, err)
+			failed++
+			continue
+		}
+		if err := os.WriteFile(filepath.Join(out, d.Name+".yaml"), d.YAML, 0o644); err != nil {
+			return err
+		}
+		cw.Printf("%s\t%s\t%d\t%d\t%d\n", name, d.Name, d.Mapped, d.Helpers, len(d.Skipped))
+	}
+	if err := errors.Join(tw.Flush(), cw.Err()); err != nil {
+		return fmt.Errorf("write summary: %w", err)
+	}
+	if failed > 0 {
+		return fmt.Errorf("%d of %d OS definitions could not be imported", failed, len(oses))
+	}
+	return nil
+}

cmd/nmon-librenms-import/main_test.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/profile"
+)
+
+func TestRunWritesLoadableDrafts(t *testing.T) {
+	root := filepath.Join("..", "..", "internal", "librenms", "testdata", "lnms")
+	out := t.TempDir()
+	if err := run([]string{"--librenms", root, "--out", out, "--os", "acme"}); err != nil {
+		t.Fatal(err)
+	}
+	b, err := os.ReadFile(filepath.Join(out, "acme.yaml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := profile.Parse("acme.yaml", b); err != nil {
+		t.Fatalf("%v\n%s", err, b)
+	}
+	if !strings.Contains(string(b), "type: unverified") {
+		t.Error("draft metrics must be unverified")
+	}
+}
+
+func TestRunErrors(t *testing.T) {
+	root := filepath.Join("..", "..", "internal", "librenms", "testdata", "lnms")
+	for _, args := range [][]string{
+		{},
+		{"--librenms", root},
+		{"--librenms", t.TempDir(), "--out", t.TempDir()},
+		{"--librenms", root, "--out", t.TempDir(), "--os", "nosuchos"},
+	} {
+		if err := run(args); err == nil {
+			t.Errorf("%v: expected an error", args)
+		}
+	}
+}

cmd/nmon/apiclient.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+// apiFlags are the flags of every command that talks to a running nmon.
+type apiFlags struct {
+	URL, Token string
+	SkipVerify bool
+}
+
+func (f *apiFlags) register(fs *flag.FlagSet) {
+	fs.StringVar(&f.URL, "api", os.Getenv("NMON_API"), "base URL of the running nmon, such as https://nmon.example.net:8443 (or set NMON_API)")
+	fs.StringVar(&f.Token, "token", os.Getenv("NMON_TOKEN"), "admin API token, nma_... (or set NMON_TOKEN)")
+	fs.BoolVar(&f.SkipVerify, "tls-skip-verify", false,
+		"do not verify the server certificate (a self-signed certificate on a trusted network)")
+}
+
+func (f *apiFlags) validate() error {
+	if f.URL == "" {
+		return errors.New("--api is required (the URL of the running nmon)")
+	}
+	if f.Token == "" {
+		return errors.New("--token is required (an admin API token; create one in Settings, or POST /api/v1/tokens)")
+	}
+	return nil
+}
+
+// client returns an HTTP client without an overall timeout: a backup download can take minutes. ctx bounds each call.
+func (f *apiFlags) client() *http.Client {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	if f.SkipVerify {
+		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12} // opt-in flag for self-signed certificates
+	}
+	return &http.Client{Transport: tr, Timeout: 0}
+}
+
+// call sends one request and returns the open response when the status is 2xx; any other status is an error
+// carrying the problem detail. The caller closes the body.
+func (f *apiFlags) call(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
+	req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(f.URL, "/")+path, body)
+	if err != nil {
+		cancel()
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+f.Token)
+	resp, err := f.client().Do(req)
+	if err != nil {
+		cancel()
+		return nil, err
+	}
+	if resp.StatusCode/100 != 2 {
+		defer cancel()
+		defer func() { _ = resp.Body.Close() }()
+		return nil, problemError(resp)
+	}
+	resp.Body = &cancelOnClose{ReadCloser: resp.Body, cancel: cancel}
+	return resp, nil
+}
+
+type cancelOnClose struct {
+	io.ReadCloser
+	cancel context.CancelFunc
+}
+
+func (c *cancelOnClose) Close() error { defer c.cancel(); return c.ReadCloser.Close() }
+
+// problemError turns an RFC 9457 response into an error with the server's explanation.
+func problemError(resp *http.Response) error {
+	b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
+	var p struct {
+		Title, Detail string
+	}
+	if json.Unmarshal(b, &p) == nil && (p.Title != "" || p.Detail != "") {
+		return fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(p.Title+": "+p.Detail))
+	}
+	return fmt.Errorf("%s", resp.Status)
+}

cmd/nmon/backup.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/v10networks/nmon/internal/backup"
+)
+
+func init() {
+	register("backup", "write a backup of inventory, metrics, config and profiles from a running nmon", backupCmd)
+}
+
+// backupCmd asks the running nmon to take its online backup and saves the stream. A running nmon owns the
+// stores, so the work happens inside it (POST /api/v1/admin/backup); this command only downloads the result.
+func backupCmd(ctx context.Context, args []string) error {
+	var af apiFlags
+	var includeKey bool
+	fs := newFlagSet("backup", "backup FILE --api URL --token TOKEN [--include-key]")
+	af.register(fs)
+	fs.BoolVar(&includeKey, "include-key", false, "put the encryption key file in the archive (protect the archive like the key)")
+	file, rest, err := leadingArg(args)
+	if err != nil {
+		return err
+	}
+	if err := fs.Parse(rest); err != nil {
+		return flagErr(err)
+	}
+	if file == "" && fs.NArg() == 1 {
+		file = fs.Arg(0)
+	}
+	if file == "" || fs.NArg() > 1 {
+		return errors.New("usage: nmon backup FILE --api URL --token TOKEN [--include-key]")
+	}
+	if err := af.validate(); err != nil {
+		return err
+	}
+	resp, err := af.call(ctx, "POST", "/api/v1/admin/backup?"+url.Values{"include_key": {strconv.FormatBool(includeKey)}}.Encode(), nil)
+	if err != nil {
+		return fmt.Errorf("backup: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	included := resp.Header.Get("X-Nmon-Key-Included") == "true"
+	n, err := saveAtomically(file, resp.Body)
+	if err != nil {
+		return fmt.Errorf("backup: %w", err)
+	}
+	fmt.Println(backup.KeyNotice(included))
+	fmt.Printf("backup: wrote %s (%d bytes)\n", file, n)
+	return nil
+}
+
+// leadingArg splits a positional first argument from the flags after it ("nmon backup FILE --api ...").
+func leadingArg(args []string) (string, []string, error) {
+	if len(args) > 0 && len(args[0]) > 0 && args[0][0] != '-' {
+		return args[0], args[1:], nil
+	}
+	return "", args, nil
+}
+
+// saveAtomically writes r to a temp file beside path, syncs it and renames it, so an interrupted download never
+// leaves a truncated file under the final name.
+func saveAtomically(path string, r io.Reader) (n int64, err error) {
+	f, err := os.CreateTemp(filepath.Dir(path), ".nmon-backup-*")
+	if err != nil {
+		return 0, err
+	}
+	defer func() {
+		if err != nil {
+			_ = os.Remove(f.Name()) // best effort: the download error is the one to report
+		}
+	}()
+	if n, err = io.Copy(f, r); err != nil {
+		_ = f.Close() // best effort: the copy error is the one to report
+		return 0, err
+	}
+	if err = f.Sync(); err != nil {
+		_ = f.Close() // best effort: the sync error is the one to report
+		return 0, err
+	}
+	if err = f.Close(); err != nil {
+		return 0, err
+	}
+	if err = os.Chmod(f.Name(), 0o600); err != nil {
+		return 0, err
+	}
+	return n, os.Rename(f.Name(), path)
+}

cmd/nmon/discover.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/v10networks/nmon/internal/cliout"
+)
+
+func init() {
+	register("discover", "run discovery for a device now (asks a running nmon)", discoverCmd)
+}
+
+// discoverReport mirrors the API's DiscoverView.
+type discoverReport struct {
+	Profile        string `json:"profile"`
+	ProfileMissing string `json:"profile_missing"`
+	Partial        bool   `json:"partial"`
+	Tables         []struct {
+		Name, Status, Detail string
+		Rows                 int
+	} `json:"tables"`
+	Matched          int      `json:"interfaces_matched"`
+	New              int      `json:"interfaces_new"`
+	Retired          int      `json:"interfaces_retired"`
+	Split            int      `json:"interfaces_split"`
+	Unretired        int      `json:"interfaces_unretired"`
+	Reviews          int      `json:"identity_reviews_opened"`
+	HardwareReplaced bool     `json:"hardware_replaced"`
+	Warnings         []string `json:"warnings"`
+}
+
+func discoverCmd(ctx context.Context, args []string) error {
+	fs := profileFlagSet("discover", "discover <slug> [--api URL]")
+	fs.Usage = func() {
+		// Usage text is best-effort; a failed write to the terminal has nowhere to go.
+		_, _ = fmt.Fprintln(os.Stderr, "usage: nmon discover <slug> [--api URL] [--data-dir DIR]\n\n"+
+			"Asks the running nmon at --api to discover the device now and prints what changed.")
+		fs.PrintDefaults()
+	}
+	var api, dataDir string
+	var timeout time.Duration
+	fs.StringVar(&api, "api", "http://[::1]:8443", "base URL of the running nmon")
+	fs.StringVar(&dataDir, "data-dir", "", "accepted for symmetry with serve; the API of the running server is used")
+	fs.DurationVar(&timeout, "timeout", 60*time.Second, "how long to wait for the discovery")
+	pos, err := parseInterspersed(fs, args)
+	if err != nil {
+		return flagErr(err)
+	}
+	if len(pos) != 1 {
+		return fmt.Errorf("usage: nmon discover <slug> [--api URL]")
+	}
+	rep, err := requestDiscover(ctx, api, pos[0], timeout)
+	if err != nil {
+		return err
+	}
+	return rep.print(os.Stdout, pos[0])
+}
+
+func requestDiscover(ctx context.Context, api, slug string, timeout time.Duration) (*discoverReport, error) {
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	u := strings.TrimRight(api, "/") + "/api/v1/devices/" + url.PathEscape(slug) + "/discover"
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, bytes.NewReader(nil))
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot reach nmon at %s (is it running? use --api): %w", api, err)
+	}
+	defer func() { _ = resp.Body.Close() }() // body fully read below; a close error changes nothing
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if resp.StatusCode != http.StatusOK {
+		var p struct{ Title, Detail string }
+		_ = json.Unmarshal(body, &p)
+		msg := p.Detail
+		if msg == "" {
+			msg = p.Title
+		}
+		return nil, fmt.Errorf("%s: %d %s", slug, resp.StatusCode, msg)
+	}
+	var rep discoverReport
+	if err := json.Unmarshal(body, &rep); err != nil {
+		return nil, err
+	}
+	return &rep, nil
+}
+
+func (r *discoverReport) print(out io.Writer, slug string) error {
+	w := cliout.New(out)
+	w.Printf("discovered %s: profile %s\n", slug, r.Profile)
+	if r.ProfileMissing != "" {
+		w.Printf("  profile %q no longer exists; generic is in use\n", r.ProfileMissing)
+	}
+	w.Printf("  interfaces: %d matched, %d new, %d retired, %d unretired, %d split\n",
+		r.Matched, r.New, r.Retired, r.Unretired, r.Split)
+	if r.Reviews > 0 {
+		w.Printf("  %d identity review(s) opened: nothing was merged automatically\n", r.Reviews)
+	}
+	if r.HardwareReplaced {
+		w.Println("  hardware replaced: chassis serial changed, history continues (review opened)")
+	}
+	for _, t := range r.Tables {
+		if t.Status != "ok" {
+			w.Printf("  table %s: %s %s\n", t.Name, t.Status, t.Detail)
+		}
+	}
+	if r.Partial {
+		w.Println("  PARTIAL: a table overran its limit; its previous result was kept")
+	}
+	for _, x := range r.Warnings {
+		w.Printf("  warning: %s\n", x)
+	}
+	return w.Err()
+}

cmd/nmon/main.go

@@ -31,8 +31,6 @@ func init() {
 	for name, summary := range map[string]string{
 		"discover": "run discovery for a device now",
 		"profile":  "test, record and draft device profiles",
-		"backup":   "write a backup of inventory, metrics, config and profiles",
-		"restore":  "restore a backup into a clean data dir",
 		"loadtest": "run the scale test against simulated devices",
 	} {
 		if _, ok := commands[name]; !ok {

cmd/nmon/migrate.go

@@ -44,7 +44,7 @@ func migrate(ctx context.Context, args []string) error {
 	if err != nil {
 		return err
 	}
-	defer st.Close()
+	defer func() { _ = st.Close() }() // migrations are committed; a close error changes nothing
 	if action == "down" {
 		if err := st.Rollback(ctx); err != nil {
 			return err

cmd/nmon/openapi.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/v10networks/nmon/internal/app"
+)
+
+func init() {
+	register("openapi", "print the generated OpenAPI 3.1 document (nmon openapi > openapi.json)", openapiCmd)
+}
+
+func openapiCmd(_ context.Context, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("openapi takes no arguments, got %q", args[0])
+	}
+	spec, err := app.OpenAPISpec()
+	if err != nil {
+		return fmt.Errorf("generate spec: %w", err)
+	}
+	_, err = fmt.Fprintf(os.Stdout, "%s\n", spec)
+	return err
+}

cmd/nmon/ops.go

@@ -0,0 +1,229 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/v10networks/nmon/internal/app"
+	"github.com/v10networks/nmon/internal/auth"
+	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/config"
+	"github.com/v10networks/nmon/internal/keyrotate"
+	"github.com/v10networks/nmon/internal/secrets"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+func init() {
+	register("key", "rotate the encryption key: nmon key rotate (nmon stopped)", keyCmd)
+	register("unlock", "clear break-glass lockouts: nmon unlock [IP|all]", unlockCmd)
+	register("secret", "store a named secret before the first start: nmon secret set NAME", secretCmd)
+	register("store", "operate the metric stores of a running nmon: nmon store restart [hot|long]", storeCmd)
+}
+
+// openInventory opens the inventory named by the shared store flags. keys may be nil for commands that never
+// decrypt (unlock).
+func openInventory(ctx context.Context, f *config.Flags, kr *secrets.Keyring) (store.Store, error) {
+	if f.DataDir == "" {
+		return nil, errors.New("--data-dir is required")
+	}
+	dsn := f.DBDSN
+	if dsn == "" {
+		if f.DBDriver != store.DriverSQLite {
+			return nil, fmt.Errorf("--db-dsn is required for --db-driver %s", f.DBDriver)
+		}
+		dsn = config.InventoryDB(f.DataDir)
+	}
+	return store.Open(ctx, f.DBDriver, dsn, kr, clock.Real())
+}
+
+// keyCmd is "nmon key rotate": a new key version for new data, every credential re-sealed under it, the old
+// key kept in the file so older backups still decrypt. nmon must be stopped: a running process holds the old keys.
+func keyCmd(ctx context.Context, args []string) error {
+	if len(args) == 0 || args[0] != "rotate" {
+		return errors.New("usage: nmon key rotate --data-dir DIR --secret-key-file FILE   (stop nmon first)")
+	}
+	var f config.Flags
+	fs := newFlagSet("key rotate", "key rotate --data-dir DIR --secret-key-file FILE [flags]")
+	storeFlags(fs, &f)
+	fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile, "the key file to rotate")
+	if err := fs.Parse(args[1:]); err != nil {
+		return flagErr(err)
+	}
+	res, err := keyrotate.Rotate(ctx, f.SecretKeyFile, func(ctx context.Context, kr *secrets.Keyring) (store.Store, error) {
+		st, err := openInventory(ctx, &f, kr)
+		if err != nil {
+			return nil, err
+		}
+		return st, st.Migrate(ctx)
+	})
+	if err != nil {
+		return err
+	}
+	fmt.Printf("key rotate: key version %d -> %d; re-encrypted %d SNMP credentials and %d secrets; "+
+		"the previous key stays in %s so older backups still restore\n",
+		res.OldVersion, res.NewVersion, res.Credentials, res.Secrets, f.SecretKeyFile)
+	return nil
+}
+
+// unlockCmd clears the break-glass lockout of one address, or of all. It works on the database, so it takes
+// effect in a running nmon too.
+func unlockCmd(ctx context.Context, args []string) error {
+	target, rest, err := leadingArg(args)
+	if err != nil {
+		return err
+	}
+	var f config.Flags
+	fs := newFlagSet("unlock", "unlock [IP|all] --data-dir DIR [flags]")
+	storeFlags(fs, &f)
+	if err := fs.Parse(rest); err != nil {
+		return flagErr(err)
+	}
+	if target == "" && fs.NArg() == 1 {
+		target = fs.Arg(0)
+	}
+	if target == "" {
+		target = "all"
+	}
+	st, err := openInventory(ctx, &f, nil)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = st.Close() }() // the unlock is committed
+	if err := st.Migrate(ctx); err != nil {
+		return err
+	}
+	n, err := auth.Unlock(ctx, st, "console", target)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("unlock: cleared %d lockout(s) for %s\n", n, target)
+	return nil
+}
+
+// storeCmd is "nmon store restart [hot|long]": resume a store the supervisor gave up on.
+func storeCmd(ctx context.Context, args []string) error {
+	if len(args) == 0 || args[0] != "restart" {
+		return errors.New("usage: nmon store restart [hot|long] --api URL --token TOKEN")
+	}
+	name, rest, err := leadingArg(args[1:])
+	if err != nil {
+		return err
+	}
+	var af apiFlags
+	fs := flag.NewFlagSet("store restart", flag.ContinueOnError)
+	af.register(fs)
+	if err := fs.Parse(rest); err != nil {
+		return flagErr(err)
+	}
+	if err := af.validate(); err != nil {
+		return err
+	}
+	names := []string{"hot", "long"}
+	if name != "" {
+		names = []string{name}
+	}
+	var errs []error
+	for _, n := range names {
+		resp, err := af.call(ctx, http.MethodPost, "/api/v1/admin/stores/"+n+"/restart", nil)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", n, err))
+			continue
+		}
+		_ = resp.Body.Close() // 204: nothing to read
+		fmt.Printf("store restart: %s store resumed\n", n)
+	}
+	return errors.Join(errs...)
+}
+
+// secretCmd is "nmon secret set NAME": it stores a named secret (the OIDC client secret, a Proxmox token) in the
+// inventory without a running nmon, reading the value from --value-file or standard input. It exists because
+// OIDC needs its client secret at start while the API that stores secrets needs a way to sign in.
+func secretCmd(ctx context.Context, args []string) error {
+	if len(args) < 2 || args[0] != "set" {
+		return errors.New("usage: nmon secret set NAME --data-dir DIR --secret-key-file FILE [--value-file FILE]   (value on stdin otherwise)")
+	}
+	name, rest, err := leadingArg(args[1:])
+	if err != nil || name == "" {
+		return errors.New("usage: nmon secret set NAME --data-dir DIR --secret-key-file FILE")
+	}
+	var f config.Flags
+	var valueFile string
+	fs := newFlagSet("secret set", "secret set NAME --data-dir DIR --secret-key-file FILE [flags]")
+	storeFlags(fs, &f)
+	fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile, "the encryption key file")
+	fs.StringVar(&valueFile, "value-file", "", "read the secret from this file instead of standard input")
+	if err := fs.Parse(rest); err != nil {
+		return flagErr(err)
+	}
+	value, err := readSecretValue(valueFile)
+	if err != nil {
+		return err
+	}
+	kr, err := keyringForSecrets(&f)
+	if err != nil {
+		return err
+	}
+	st, err := openInventory(ctx, &f, kr)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = st.Close() }() // the secret is committed
+	if err := st.Migrate(ctx); err != nil {
+		return err
+	}
+	if err := st.PutSecret(ctx, name, value); err != nil {
+		return err
+	}
+	if err := st.AppendAudit(ctx, "console", "secret.put", "secret:"+name, nil, nil); err != nil { // the value is never audited
+		return err
+	}
+	fmt.Printf("secret set: stored %q (encrypted with key version %d)\n", name, kr.CurrentVersion())
+	return nil
+}
+
+// readSecretValue reads the secret from a file or standard input, without its trailing newline.
+func readSecretValue(valueFile string) (string, error) {
+	var src io.Reader = os.Stdin
+	if valueFile != "" {
+		vf, err := os.Open(valueFile)
+		if err != nil {
+			return "", err
+		}
+		defer func() { _ = vf.Close() }() // read-only
+		src = vf
+	}
+	raw, err := io.ReadAll(io.LimitReader(src, 4097))
+	if err != nil {
+		return "", err
+	}
+	value := strings.TrimRight(string(raw), "\r\n")
+	if value == "" || len(value) > 4096 {
+		return "", errors.New("the secret must be 1 to 4096 bytes")
+	}
+	return value, nil
+}
+
+// keyringForSecrets loads the key file. On a brand-new install (no key file and no inventory yet) it creates the
+// key first, as `nmon serve` would, so the OIDC client secret can be stored before the first start.
+func keyringForSecrets(f *config.Flags) (*secrets.Keyring, error) {
+	if _, err := os.Stat(f.SecretKeyFile); errors.Is(err, os.ErrNotExist) {
+		if _, err := os.Stat(config.InventoryDB(f.DataDir)); err == nil {
+			return nil, fmt.Errorf("secret key file %s is missing but the inventory exists; restore the key file", f.SecretKeyFile)
+		}
+		if err := secrets.GenerateKeyFile(f.SecretKeyFile); err != nil {
+			return nil, fmt.Errorf("generate secret key: %w", err)
+		}
+		fmt.Printf("secret set: created a new encryption key at %s; back it up now\n", f.SecretKeyFile)
+	}
+	kr, err := secrets.LoadKeyring(f.SecretKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("secret key: %w", err)
+	}
+	return kr, nil
+}

cmd/nmon/profile.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+func init() {
+	register("profile", "test, record and draft device profiles", profileCmd)
+}
+
+const profileUsage = `usage: nmon profile <subcommand> [flags]
+
+subcommands:
+  new      compile MIB objects into draft profile metrics (--mib MODULE)
+  record   read-only SNMP walk of a device into a redacted .snmprec file
+  test     discovery plus one poll with the matching profile
+`
+
+func profileCmd(ctx context.Context, args []string) error {
+	if len(args) == 0 || args[0] == "-h" || args[0] == "--help" || args[0] == "help" {
+		fmt.Fprint(os.Stderr, profileUsage)
+		if len(args) == 0 {
+			os.Exit(2)
+		}
+		return nil
+	}
+	switch args[0] {
+	case "new":
+		return profileNew(ctx, args[1:], os.Stdout)
+	case "record":
+		return profileRecord(ctx, args[1:], os.Stdout)
+	case "test":
+		return profileTest(ctx, args[1:], os.Stdout)
+	}
+	fmt.Fprint(os.Stderr, profileUsage)
+	return fmt.Errorf("profile: unknown subcommand %q", args[0])
+}
+
+// listFlag is a repeatable string flag that also accepts comma-separated values.
+type listFlag []string
+
+func (l *listFlag) String() string { return strings.Join(*l, ",") }
+func (l *listFlag) Set(v string) error {
+	for _, s := range strings.Split(v, ",") {
+		if s = strings.TrimSpace(s); s != "" {
+			*l = append(*l, s)
+		}
+	}
+	return nil
+}
+
+// parseInterspersed parses flags that may come before or after positional arguments.
+func parseInterspersed(fs *flag.FlagSet, args []string) ([]string, error) {
+	var pos []string
+	for {
+		if err := fs.Parse(args); err != nil {
+			return nil, err
+		}
+		args = fs.Args()
+		if len(args) == 0 {
+			return pos, nil
+		}
+		pos = append(pos, args[0])
+		args = args[1:]
+	}
+}
+
+// profileFlagSet returns a FlagSet that prints its usage to stderr.
+func profileFlagSet(name, usage string) *flag.FlagSet {
+	fs := newFlagSet("profile "+name, usage)
+	fs.SetOutput(io.Writer(os.Stderr))
+	return fs
+}

cmd/nmon/profile_new.go

@@ -0,0 +1,483 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/v10networks/nmon/internal/mib"
+)
+
+const defaultSystemMIBs = "/usr/share/nmon/mibs"
+
+// mibSearchPath returns the directories to read MIBs from: explicit --mibs-dir
+// values, else <data-dir>/mibs followed by the system location.
+func mibSearchPath(explicit []string, dataDir string) ([]string, error) {
+	if len(explicit) > 0 {
+		for _, d := range explicit {
+			if st, err := os.Stat(d); err != nil || !st.IsDir() {
+				return nil, fmt.Errorf("--mibs-dir %s: not a directory", d)
+			}
+		}
+		return explicit, nil
+	}
+	var out []string
+	var cands []string
+	if dataDir != "" {
+		cands = append(cands, filepath.Join(dataDir, "mibs"))
+	}
+	cands = append(cands, defaultSystemMIBs)
+	for _, d := range cands {
+		if st, err := os.Stat(d); err == nil && st.IsDir() {
+			out = append(out, d)
+		}
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("no MIB directory found (looked in %s); pass --mibs-dir", strings.Join(cands, ", "))
+	}
+	return out, nil
+}
+
+func profileNew(_ context.Context, args []string, stdout io.Writer) error {
+	fs := profileFlagSet("new",
+		"profile new --mib MODULE [--table NAME] [--objects a,b] [--mibs-dir DIR]... [--data-dir DIR] [--out FILE]")
+	var (
+		module, tableName, out, dataDir string
+		objects, mibsDirs               listFlag
+	)
+	fs.StringVar(&module, "mib", "", "MIB module to compile, e.g. IF-MIB (required)")
+	fs.StringVar(&tableName, "table", "", "compile every readable column of this table")
+	fs.Var(&objects, "objects", "comma-separated object names (NAME or MODULE::NAME); repeatable")
+	fs.Var(&mibsDirs, "mibs-dir",
+		"directory of MIB files, searched recursively; repeatable (default <data-dir>/mibs, then "+defaultSystemMIBs+")")
+	fs.StringVar(&dataDir, "data-dir", "", "nmon data folder; its mibs/ subfolder is searched")
+	fs.StringVar(&out, "out", "", "write the draft here instead of stdout")
+	rest, err := parseInterspersed(fs, args)
+	if err != nil {
+		return flagErr(err)
+	}
+	if len(rest) > 0 {
+		return fmt.Errorf("unexpected argument %q", rest[0])
+	}
+	if module == "" {
+		return fmt.Errorf("--mib is required")
+	}
+	path, err := mibSearchPath(mibsDirs, dataDir)
+	if err != nil {
+		return err
+	}
+	l := mib.NewLoader(path...)
+	draft, err := draftProfile(l, draftSpec{Module: module, Table: tableName, Objects: objects})
+	if err != nil {
+		return err
+	}
+	if out != "" {
+		return os.WriteFile(out, draft, 0o644)
+	}
+	_, err = stdout.Write(draft)
+	return err
+}
+
+type draftSpec struct {
+	Module  string
+	Table   string
+	Objects []string
+}
+
+// unitMap maps UNITS clauses (lower case) to the profile unit vocabulary, with
+// the scale that converts the MIB's unit to it.
+var unitMap = map[string]struct {
+	unit  string
+	scale float64
+}{
+	"seconds": {"seconds", 0}, "second": {"seconds", 0}, "secs": {"seconds", 0}, "sec": {"seconds", 0},
+	"minutes": {"minutes", 0}, "minute": {"minutes", 0},
+	"degrees celsius": {"celsius", 0}, "degrees centigrade": {"celsius", 0}, "celsius": {"celsius", 0},
+	"degrees c": {"celsius", 0}, "deg c": {"celsius", 0},
+	"volts": {"volts", 0}, "volt": {"volts", 0}, "millivolts": {"volts", 0.001}, "mv": {"volts", 0.001},
+	"amperes": {"amps", 0}, "amps": {"amps", 0}, "amp": {"amps", 0}, "milliamperes": {"amps", 0.001},
+	"milliamps": {"amps", 0.001}, "ma": {"amps", 0.001},
+	"watts": {"watts", 0}, "watt": {"watts", 0}, "milliwatts": {"watts", 0.001}, "mw": {"watts", 0.001},
+	"dbm": {"dbm", 0},
+	"rpm": {"rpm", 0}, "revolutions per minute": {"rpm", 0},
+	"percent": {"percent", 0}, "%": {"percent", 0}, "percentage": {"percent", 0},
+	"bytes": {"bytes", 0}, "octets": {"octets", 0}, "kilobytes": {"bytes", 1024}, "megabytes": {"bytes", 1048576},
+	"bits": {"bits", 0}, "bits per second": {"bps", 0}, "bps": {"bps", 0}, "kilobits per second": {"bps", 1000},
+	"packets": {"packets", 0}, "errors": {"errors", 0},
+	"hertz": {"hertz", 0}, "hz": {"hertz", 0},
+}
+
+var hintRE = regexp.MustCompile(`^d-([0-9]+)$`)
+
+type draftMetric struct {
+	obj      *mib.Object
+	label    string // index_label column
+	labelAs  string
+	comments []string
+	typ      string
+	unit     string
+	scale    float64
+	enum     map[int]string
+}
+
+// draftProfile compiles the selected objects into a complete profile document
+// whose unit and interval are TODO so that profile.Parse refuses it until a human fills them in.
+func draftProfile(l *mib.Loader, spec draftSpec) ([]byte, error) {
+	m, err := l.Load(spec.Module)
+	if err != nil {
+		return nil, err
+	}
+	objs, err := selectObjects(l, m, spec)
+	if err != nil {
+		return nil, err
+	}
+	return []byte(renderDraft(spec, m, draftMetrics(l, m, objs))), nil
+}
+
+// selectObjects picks the objects to compile: a table's readable columns, the
+// named objects, or (when neither is given) every readable scalar and column.
+// The result is sorted by OID.
+func selectObjects(l *mib.Loader, m *mib.Module, spec draftSpec) ([]*mib.Object, error) {
+	chosen := map[string]*mib.Object{}
+	add := func(o *mib.Object) { chosen[o.Module+"::"+o.Name] = o }
+	if spec.Table != "" {
+		cols, err := tableColumns(m, spec)
+		if err != nil {
+			return nil, err
+		}
+		for _, o := range cols {
+			add(o)
+		}
+	}
+	named, err := namedObjects(l, spec)
+	if err != nil {
+		return nil, err
+	}
+	for _, o := range named {
+		add(o)
+	}
+	if spec.Table == "" && len(spec.Objects) == 0 {
+		for _, o := range allReadable(m) {
+			add(o)
+		}
+	}
+	if len(chosen) == 0 {
+		return nil, fmt.Errorf("%s: no readable scalar or column objects selected", spec.Module)
+	}
+	objs := make([]*mib.Object, 0, len(chosen))
+	for _, o := range chosen {
+		objs = append(objs, o)
+	}
+	sort.Slice(objs, func(i, j int) bool { return oidLess(objs[i].OID, objs[j].OID) })
+	return objs, nil
+}
+
+// allReadable lists every current, readable scalar and column of m.
+func allReadable(m *mib.Module) []*mib.Object {
+	var out []*mib.Object
+	for _, o := range m.Objects {
+		if (o.Kind == mib.KindScalar || o.Kind == mib.KindColumn) && readable(o) && o.Status != "obsolete" && o.OID != "" {
+			out = append(out, o)
+		}
+	}
+	return out
+}
+
+// tableColumns lists the readable columns of spec.Table.
+func tableColumns(m *mib.Module, spec draftSpec) ([]*mib.Object, error) {
+	tb := m.Lookup(spec.Table)
+	if tb == nil || tb.Kind != mib.KindTable {
+		return nil, fmt.Errorf("%s: no table %q (tables are %s)", spec.Module, spec.Table, strings.Join(tableNames(m), ", "))
+	}
+	var cols []*mib.Object
+	for _, o := range m.Objects {
+		if o.Kind == mib.KindColumn && o.Table == spec.Table && readable(o) {
+			cols = append(cols, o)
+		}
+	}
+	return cols, nil
+}
+
+// namedObjects resolves --objects entries; a bare NAME belongs to spec.Module.
+func namedObjects(l *mib.Loader, spec draftSpec) ([]*mib.Object, error) {
+	var out []*mib.Object
+	for _, n := range spec.Objects {
+		q := n
+		if !strings.Contains(q, "::") {
+			q = spec.Module + "::" + n
+		}
+		o, err := l.Object(q)
+		if err != nil {
+			return nil, err
+		}
+		if o.Kind != mib.KindScalar && o.Kind != mib.KindColumn {
+			return nil, fmt.Errorf("%s is a %s, not a scalar or table column", q, o.Kind)
+		}
+		out = append(out, o)
+	}
+	return out, nil
+}
+
+// draftMetrics converts objs to draft metrics. Label columns used by
+// index_label must exist as metrics of the profile, so the ones not already
+// selected are appended, sorted by OID.
+func draftMetrics(l *mib.Loader, m *mib.Module, objs []*mib.Object) []draftMetric {
+	labels := map[string]*mib.Object{}
+	var metrics []draftMetric
+	have := map[string]bool{}
+	for _, o := range objs {
+		have[o.Name] = true
+	}
+	for _, o := range objs {
+		dm := convert(o)
+		if o.Kind == mib.KindColumn {
+			if lo := setLabel(&dm, l, m, o); lo != nil && !have[lo.Name] {
+				labels[lo.Name] = lo
+			}
+		}
+		metrics = append(metrics, dm)
+	}
+	var extra []*mib.Object
+	for _, o := range labels {
+		extra = append(extra, o)
+	}
+	sort.Slice(extra, func(i, j int) bool { return oidLess(extra[i].OID, extra[j].OID) })
+	for _, o := range extra {
+		dm := convert(o)
+		dm.comments = append(dm.comments, "added as the index_label column of the metrics above")
+		metrics = append(metrics, dm)
+	}
+	return metrics
+}
+
+// setLabel fills dm's index_label from labelFor. It returns the label column
+// when the profile must carry it as a metric of its own (a component label), else nil.
+func setLabel(dm *draftMetric, l *mib.Loader, m *mib.Module, o *mib.Object) *mib.Object {
+	col, as, note := labelFor(l, m, o)
+	dm.label, dm.labelAs = col, as
+	if note != "" {
+		dm.comments = append(dm.comments, note)
+	}
+	if as != "component" {
+		return nil
+	}
+	return m.Lookup(col)
+}
+
+// renderDraft writes the profile YAML document.
+func renderDraft(spec draftSpec, m *mib.Module, metrics []draftMetric) string {
+	var b strings.Builder
+	name := "draft-" + strings.ToLower(spec.Module)
+	fmt.Fprintf(&b, "# DRAFT compiled by `nmon profile new` from %s (%s).\n", spec.Module, filepath.Base(m.File))
+	fmt.Fprintf(&b, "# Save as %s.yaml. Replace every TODO (unit, interval, index_label) and add a match: section.\n", name)
+	b.WriteString("# The loader rejects TODO on purpose, so an unreviewed draft can never be polled.\n")
+	fmt.Fprintf(&b, "name: %s\nextends: generic\nprovenance: %q\nmetrics:\n", name,
+		"drafted by nmon profile new from "+spec.Module+"; review before use")
+	for _, dm := range metrics {
+		writeDraftMetric(&b, dm)
+	}
+	return b.String()
+}
+
+func tableNames(m *mib.Module) []string {
+	var out []string
+	for _, o := range m.Objects {
+		if o.Kind == mib.KindTable {
+			out = append(out, o.Name)
+		}
+	}
+	return out
+}
+
+func readable(o *mib.Object) bool {
+	switch o.Access {
+	case "read-only", "read-write", "read-create", "mandatory", "":
+		return o.Access != "" || o.Kind == mib.KindScalar
+	}
+	return false
+}
+
+func oidLess(a, b string) bool {
+	as, bs := strings.Split(a, "."), strings.Split(b, ".")
+	for i := 0; i < len(as) && i < len(bs); i++ {
+		x, _ := strconv.Atoi(as[i])
+		y, _ := strconv.Atoi(bs[i])
+		if x != y {
+			return x < y
+		}
+	}
+	return len(as) < len(bs)
+}
+
+// convert maps one MIB object to a draft metric.
+func convert(o *mib.Object) draftMetric {
+	dm := draftMetric{obj: o, unit: "TODO"}
+	if o.TC != "" || o.DisplayHint != "" {
+		dm.comments = append(dm.comments, conventionNote(o))
+	}
+	applyBaseType(&dm, o)
+	if o.Units != "" {
+		if u, ok := unitMap[strings.ToLower(o.Units)]; ok && dm.typ != "status" && dm.typ != "text" {
+			dm.unit, dm.scale = u.unit, u.scale
+		} else {
+			dm.comments = append(dm.comments, "UNITS "+strconv.Quote(o.Units)+" is not in the unit vocabulary")
+		}
+	}
+	return dm
+}
+
+// conventionNote describes the textual convention and DISPLAY-HINT, and
+// suggests a scale when the hint implies decimal places ("d-2").
+func conventionNote(o *mib.Object) string {
+	c := "textual convention " + o.TC
+	if o.TC == "" {
+		c = "no textual convention"
+	}
+	if o.DisplayHint != "" {
+		c += ", DISPLAY-HINT " + strconv.Quote(o.DisplayHint)
+	}
+	if mm := hintRE.FindStringSubmatch(o.DisplayHint); mm != nil {
+		n, _ := strconv.Atoi(mm[1])
+		s := "0." + strings.Repeat("0", n-1) + "1"
+		if n == 0 {
+			s = "1"
+		}
+		c += "; the value has " + mm[1] + " implied decimal place(s), suggest scale: " + s
+	}
+	return c
+}
+
+// applyBaseType sets the metric type (and unit where the type implies one)
+// from the object's SMI base type.
+func applyBaseType(dm *draftMetric, o *mib.Object) {
+	switch o.BaseType {
+	case mib.BaseCounter64:
+		dm.typ = "counter64"
+	case mib.BaseCounter32:
+		dm.typ = "counter32"
+	case mib.BaseGauge32, mib.BaseUnsigned32, mib.BaseInteger32, mib.BaseInteger:
+		if len(o.Enum) > 0 {
+			dm.typ, dm.unit, dm.enum = "status", "enum", o.Enum
+		} else {
+			dm.typ = "gauge"
+		}
+	case mib.BaseTimeTicks:
+		dm.typ, dm.unit = "gauge", "seconds"
+		dm.comments = append(dm.comments, "TimeTicks: the poller converts to seconds")
+	case mib.BaseOctets:
+		dm.typ, dm.unit = "text", "text"
+	case mib.BaseOID, mib.BaseIPAddress, mib.BaseOpaque, mib.BaseBits:
+		dm.typ, dm.unit = "text", "text"
+		dm.comments = append(dm.comments, o.BaseType+" is collected as text")
+	default:
+		dm.typ = "gauge"
+		dm.comments = append(dm.comments, fmt.Sprintf("base type of %q could not be determined; check the type", o.TC))
+	}
+}
+
+var labelNameRE = regexp.MustCompile(`(?i)(descr|name|label|alias)`)
+
+// labelFor picks the index_label for a table column: ifIndex-indexed tables are
+// labelled by interface; otherwise the first DisplayString-like column of the table.
+func labelFor(l *mib.Loader, m *mib.Module, o *mib.Object) (col, as, note string) {
+	if indexedByIfIndex(l, m, o) {
+		return "ifIndex", "iface", ""
+	}
+	if best := bestLabelColumn(m, o); best != nil {
+		return best.Name, "component", ""
+	}
+	return "TODO", "component", "no DisplayString column found in the table: set index_label.column"
+}
+
+// indexedByIfIndex follows AUGMENTS (up to 4 rows deep) looking for ifIndex in the row's INDEX.
+func indexedByIfIndex(l *mib.Loader, m *mib.Module, o *mib.Object) bool {
+	row := rowOf(m, o)
+	for depth := 0; row != nil && depth < 4; depth++ {
+		if slices.Contains(row.Index, "ifIndex") {
+			return true
+		}
+		if row.Augments == "" {
+			break
+		}
+		next := m.Lookup(row.Augments) // AUGMENTS: the row shares the augmented row's index
+		if next == nil {
+			next, _ = l.Object(row.Augments)
+		}
+		row = next
+	}
+	return false
+}
+
+// bestLabelColumn is the lowest-OID readable string column of o's table that
+// looks like a name (DisplayString, a text DISPLAY-HINT or a descr/name/label/alias name).
+func bestLabelColumn(m *mib.Module, o *mib.Object) *mib.Object {
+	var best *mib.Object
+	for _, c := range m.Objects {
+		if c.Kind != mib.KindColumn || c.Table != o.Table || !readable(c) || c.BaseType != mib.BaseOctets {
+			continue
+		}
+		if c.TC != "DisplayString" && !strings.HasSuffix(c.DisplayHint, "a") && !labelNameRE.MatchString(c.Name) {
+			continue
+		}
+		if best == nil || oidLess(c.OID, best.OID) {
+			best = c
+		}
+	}
+	return best
+}
+
+func rowOf(m *mib.Module, col *mib.Object) *mib.Object {
+	i := strings.LastIndexByte(col.OID, '.')
+	if i < 0 {
+		return nil
+	}
+	for _, o := range m.Objects {
+		if o.Kind == mib.KindRow && o.OID == col.OID[:i] {
+			return o
+		}
+	}
+	return nil
+}
+
+func writeDraftMetric(b *strings.Builder, dm draftMetric) {
+	o := dm.obj
+	oid := o.OID
+	if o.Kind == mib.KindScalar {
+		oid += ".0"
+	}
+	for _, c := range dm.comments {
+		fmt.Fprintf(b, "  # %s\n", c)
+	}
+	if o.Description != "" {
+		d := o.Description
+		if len(d) > 100 {
+			d = d[:100] + "..."
+		}
+		fmt.Fprintf(b, "  # %s\n", d)
+	}
+	fmt.Fprintf(b, "  - name: %s\n    oid: %s\n    type: %s\n    unit: %s\n    interval: TODO\n", o.Name, oid, dm.typ, dm.unit)
+	if dm.scale != 0 {
+		fmt.Fprintf(b, "    scale: %s\n", strconv.FormatFloat(dm.scale, 'f', -1, 64))
+	}
+	if dm.label != "" {
+		fmt.Fprintf(b, "    index_label: { column: %s, as: %s }\n", dm.label, dm.labelAs)
+	}
+	if len(dm.enum) > 0 {
+		b.WriteString("    enum:\n")
+		keys := make([]int, 0, len(dm.enum))
+		for k := range dm.enum {
+			keys = append(keys, k)
+		}
+		sort.Ints(keys)
+		for _, k := range keys {
+			fmt.Fprintf(b, "      %d: %q\n", k, dm.enum[k])
+		}
+	}
+}

cmd/nmon/profile_probe.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/netip"
+	"time"
+
+	"github.com/v10networks/nmon/internal/config"
+	"github.com/v10networks/nmon/internal/profile"
+	"github.com/v10networks/nmon/internal/profiletest"
+	"github.com/v10networks/nmon/internal/snmp"
+)
+
+// resolveCredential normalises --version (v1, v2c, v3 or bare; default by
+// whether a v3 user was given) into cred.Version and checks the secrets that
+// version needs.
+func resolveCredential(version string, cred *snmp.Credential) error {
+	switch version {
+	case "":
+		version = "2c"
+		if cred.V3User != "" {
+			version = "3"
+		}
+	case "v1", "v2c", "v3":
+		version = version[1:]
+	}
+	cred.Version = version
+	switch {
+	case version != "1" && version != "2c" && version != "3":
+		return errors.New("--version must be v1, v2c or v3")
+	case version != "3" && cred.Community == "":
+		return fmt.Errorf("--community is required for SNMP v%s", version)
+	case version == "3" && cred.V3User == "":
+		return errors.New("--v3-user is required for SNMPv3")
+	}
+	return nil
+}
+
+// profileTest implements `nmon profile test`: discovery plus one poll of every
+// interval group with the matching profile, printing each metric.
+func profileTest(ctx context.Context, args []string, stdout io.Writer) error {
+	fs := profileFlagSet("test",
+		"profile test <ip> [--port N] [--version 1|2c|3] [--community C | --v3-user U ...] [--data-dir DIR]")
+	var (
+		port    int
+		version string
+		cred    snmp.Credential
+		dataDir string
+		timeout time.Duration
+	)
+	fs.IntVar(&port, "port", 161, "device UDP port")
+	fs.StringVar(&version, "version", "", "SNMP version: v1, v2c or v3 (default v3 when --v3-user is set, else v2c)")
+	fs.StringVar(&cred.Community, "community", "", "v1/v2c community")
+	fs.StringVar(&cred.V3User, "v3-user", "", "SNMPv3 user name")
+	fs.StringVar(&cred.V3AuthProto, "v3-auth-proto", "", "SNMPv3 auth protocol: MD5, SHA, SHA224, SHA256, SHA384, SHA512 (default SHA256)")
+	fs.StringVar(&cred.V3AuthPass, "v3-auth-pass", "", "SNMPv3 authentication passphrase")
+	fs.StringVar(&cred.V3PrivProto, "v3-priv-proto", "", "SNMPv3 privacy protocol: DES, AES, AES192, AES256 (default AES)")
+	fs.StringVar(&cred.V3PrivPass, "v3-priv-pass", "", "SNMPv3 privacy passphrase")
+	fs.StringVar(&cred.V3Context, "v3-context", "", "SNMPv3 context name")
+	fs.StringVar(&dataDir, "data-dir", "", "load operator profiles from <data-dir>/profiles over the shipped ones")
+	fs.DurationVar(&timeout, "timeout", 0, "per-request SNMP timeout (default 2s)")
+	pos, err := parseInterspersed(fs, args)
+	if err != nil {
+		return flagErr(err)
+	}
+	if len(pos) != 1 {
+		return fmt.Errorf("usage: nmon profile test <ip> [flags]")
+	}
+	ip, err := netip.ParseAddr(pos[0])
+	if err != nil {
+		return fmt.Errorf("bad address %q: %w", pos[0], err)
+	}
+	if err := resolveCredential(version, &cred); err != nil {
+		return err
+	}
+	dir := ""
+	if dataDir != "" {
+		dir = config.ProfilesDir(dataDir)
+	}
+	set, err := profile.Load(dir)
+	if err != nil {
+		return err
+	}
+	res, err := profiletest.Run(ctx, profiletest.Options{
+		Addr: netip.AddrPortFrom(ip, uint16(port)), Cred: cred, Profiles: set, Out: stdout, Timeout: timeout,
+	})
+	if err != nil {
+		return err
+	}
+	if res.Poll.Mismatches > 0 || res.Poll.WalkPDUs > 0 {
+		return fmt.Errorf("%d type mismatches, %d walk PDUs on the poll path", res.Poll.Mismatches, res.Poll.WalkPDUs)
+	}
+	return nil
+}

cmd/nmon/profile_record.go

@@ -0,0 +1,178 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/netip"
+	"os"
+	"time"
+
+	"github.com/gosnmp/gosnmp"
+
+	"github.com/v10networks/nmon/internal/snmp"
+	"github.com/v10networks/nmon/internal/snmprec"
+)
+
+// defaultRecordSubtrees is what `profile record` walks when --subtree is not given:
+// system, interfaces, ifMIB, host, entity, entitySensor, UPS, printer and enterprises.
+var defaultRecordSubtrees = []string{
+	"1.3.6.1.2.1.1",  // system
+	"1.3.6.1.2.1.2",  // interfaces
+	"1.3.6.1.2.1.31", // ifMIB
+	"1.3.6.1.2.1.25", // host
+	"1.3.6.1.2.1.47", // entity
+	"1.3.6.1.2.1.99", // entitySensor
+	"1.3.6.1.2.1.33", // upsMIB
+	"1.3.6.1.2.1.43", // printmib
+	"1.3.6.1.4.1",    // enterprises
+}
+
+const redacted = "REDACTED"
+
+var redactedOIDs = map[string]bool{
+	"1.3.6.1.2.1.1.4.0": true, // sysContact
+	"1.3.6.1.2.1.1.6.0": true, // sysLocation
+}
+
+type recordOpts struct {
+	Target     snmp.Target
+	Subtrees   []string
+	MaxReps    int
+	MaxRecords int
+}
+
+func profileRecord(ctx context.Context, args []string, stdout io.Writer) error {
+	fs := profileFlagSet("record",
+		"profile record <ip> [--port N] [--community C | --version 3 --v3-user U ...] [--out FILE.snmprec] [--subtree OID]...")
+	var (
+		port       int
+		version    string
+		cred       snmp.Credential
+		out        string
+		subtrees   listFlag
+		timeout    time.Duration
+		maxReps    int
+		maxRecords int
+	)
+	fs.IntVar(&port, "port", 161, "device UDP port")
+	fs.StringVar(&version, "version", "", "SNMP version: 1, 2c or 3 (default 3 when --v3-user is set, else 2c)")
+	fs.StringVar(&cred.Community, "community", "", "v1/v2c community")
+	fs.StringVar(&cred.V3User, "v3-user", "", "SNMPv3 user name")
+	fs.StringVar(&cred.V3AuthProto, "v3-auth-proto", "", "SNMPv3 auth protocol: MD5, SHA, SHA224, SHA256, SHA384, SHA512 (default SHA256)")
+	fs.StringVar(&cred.V3AuthPass, "v3-auth-pass", "", "SNMPv3 authentication passphrase")
+	fs.StringVar(&cred.V3PrivProto, "v3-priv-proto", "", "SNMPv3 privacy protocol: DES, AES, AES192, AES256, AES192C, AES256C (default AES)")
+	fs.StringVar(&cred.V3PrivPass, "v3-priv-pass", "", "SNMPv3 privacy passphrase")
+	fs.StringVar(&cred.V3Context, "v3-context", "", "SNMPv3 context name")
+	fs.StringVar(&out, "out", "", "write the recording here instead of stdout")
+	fs.Var(&subtrees, "subtree",
+		"OID to walk; repeatable (default: system, interfaces, ifMIB, host, entity, entitySensor, ups, printer, enterprises)")
+	fs.DurationVar(&timeout, "timeout", 3*time.Second, "per-request timeout")
+	fs.IntVar(&maxReps, "max-repetitions", 25, "GETBULK max-repetitions")
+	fs.IntVar(&maxRecords, "max-records", 500000, "stop with an error after this many varbinds")
+	pos, err := parseInterspersed(fs, args)
+	if err != nil {
+		return flagErr(err)
+	}
+	if len(pos) != 1 {
+		return fmt.Errorf("usage: nmon profile record <ip> [flags]")
+	}
+	ip, err := netip.ParseAddr(pos[0])
+	if err != nil {
+		return fmt.Errorf("bad address %q: %w", pos[0], err)
+	}
+	switch {
+	case version != "":
+	case cred.V3User != "":
+		version = "3"
+	default:
+		version = "2c"
+	}
+	cred.Version = version
+	if version != "3" && cred.Community == "" {
+		return fmt.Errorf("--community is required for SNMP v%s", version)
+	}
+	if version == "3" && cred.V3User == "" {
+		return fmt.Errorf("--v3-user is required for SNMPv3")
+	}
+	if len(subtrees) == 0 {
+		subtrees = defaultRecordSubtrees
+	}
+	var buf bytes.Buffer
+	n, err := recordDevice(ctx, recordOpts{
+		Target:     snmp.Target{Addr: netip.AddrPortFrom(ip, uint16(port)), Cred: cred, Timeout: timeout},
+		Subtrees:   subtrees,
+		MaxReps:    maxReps,
+		MaxRecords: maxRecords,
+	}, &buf)
+	if err != nil {
+		return err
+	}
+	if out != "" {
+		if err := os.WriteFile(out, buf.Bytes(), 0o644); err != nil {
+			return err
+		}
+		fmt.Fprintf(os.Stderr, "recorded %d varbinds to %s\n", n, out)
+		return nil
+	}
+	_, err = stdout.Write(buf.Bytes())
+	return err
+}
+
+// recordDevice walks the subtrees read-only (GETBULK, or GETNEXT on v1) and writes
+// an snmprec recording to w with the community, sysContact and sysLocation redacted.
+// It returns the number of records written.
+func recordDevice(ctx context.Context, o recordOpts, w io.Writer) (int, error) {
+	s, err := snmp.Dial(ctx, o.Target, "discovery")
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = s.Close() }() // read-only session
+	var recs []snmprec.Record
+	for _, root := range o.Subtrees {
+		err := s.Walk(ctx, root, o.MaxReps, func(vb snmp.Varbind) error {
+			if vb.Exception() {
+				return nil
+			}
+			if len(recs) >= o.MaxRecords {
+				return fmt.Errorf("more than %d varbinds; narrow the walk with --subtree", o.MaxRecords)
+			}
+			recs = append(recs, redact(vb, o.Target.Cred.Community))
+			return nil
+		})
+		if err != nil {
+			return 0, fmt.Errorf("walk %s: %w", root, err)
+		}
+	}
+	rec := snmprec.NewRecording(recs)
+	if rec.Len() == 0 {
+		return 0, fmt.Errorf("the device returned nothing under %v", o.Subtrees)
+	}
+	if err := snmprec.Write(w, rec); err != nil {
+		return 0, err
+	}
+	return rec.Len(), nil
+}
+
+// redact turns a varbind into a recording row, replacing secrets with REDACTED.
+func redact(vb snmp.Varbind, community string) snmprec.Record {
+	r := snmprec.Record{OID: vb.OID, Type: vb.Type, Value: vb.Value}
+	if redactedOIDs[vb.OID] {
+		r.Type, r.Value = gosnmp.OctetString, []byte(redacted)
+		return r
+	}
+	if community != "" {
+		switch v := vb.Value.(type) {
+		case []byte:
+			if string(v) == community {
+				r.Value = []byte(redacted)
+			}
+		case string:
+			if vb.Type == gosnmp.OctetString && v == community {
+				r.Value = []byte(redacted)
+			}
+		}
+	}
+	return r
+}

cmd/nmon/profile_test.go

@@ -0,0 +1,242 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"net/netip"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/profile"
+	"github.com/v10networks/nmon/internal/simagent"
+	"github.com/v10networks/nmon/internal/snmprec"
+)
+
+func bundledMIBs(t *testing.T) string {
+	t.Helper()
+	d, _ := filepath.Abs(filepath.Join("..", "..", "mibs"))
+	if _, err := os.Stat(d); err != nil {
+		t.Skip("bundled MIBs not present")
+	}
+	return d
+}
+
+func fillTODOs(s string) string {
+	s = strings.ReplaceAll(s, "interval: TODO", "interval: 5m")
+	return strings.ReplaceAll(s, "unit: TODO", "unit: count")
+}
+
+func runNew(t *testing.T, args ...string) string {
+	t.Helper()
+	var out bytes.Buffer
+	if err := profileNew(context.Background(), args, &out); err != nil {
+		t.Fatal(err)
+	}
+	return out.String()
+}
+
+func TestProfileNewIFMIBTable(t *testing.T) {
+	d := runNew(t, "--mib", "IF-MIB", "--table", "ifXTable", "--mibs-dir", bundledMIBs(t))
+	if _, err := profile.Parse("draft-if-mib.yaml", []byte(d)); err == nil || !strings.Contains(err.Error(), "TODO") {
+		t.Fatalf("an unfilled draft must be rejected naming TODO, got %v", err)
+	}
+	p, err := profile.Parse("draft-if-mib.yaml", []byte(fillTODOs(d)))
+	if err != nil {
+		t.Fatalf("filled draft: %v\n%s", err, d)
+	}
+	byName := map[string]profile.Metric{}
+	for _, m := range p.Metrics {
+		byName[m.Name] = m
+	}
+	hc := byName["ifHCInOctets"]
+	if hc.OID != "1.3.6.1.2.1.31.1.1.1.6" || hc.Type != profile.Counter64 || hc.IndexLabel == nil || hc.IndexLabel.Column != "ifIndex" || hc.IndexLabel.As != "iface" {
+		t.Errorf("ifHCInOctets = %+v", hc)
+	}
+	if m := byName["ifLinkUpDownTrapEnable"]; m.Type != profile.Status || m.Enum[1] != "enabled" {
+		t.Errorf("ifLinkUpDownTrapEnable = %+v", m)
+	}
+	if m := byName["ifAlias"]; m.Type != profile.Text {
+		t.Errorf("ifAlias = %+v", m)
+	}
+	if !strings.Contains(d, "DISPLAY-HINT") {
+		t.Errorf("expected a DISPLAY-HINT comment:\n%s", d)
+	}
+}
+
+func TestProfileNewScalarsAndObjects(t *testing.T) {
+	d := runNew(t, "--mib", "SNMPv2-MIB", "--objects", "sysUpTime,sysName", "--mibs-dir", bundledMIBs(t))
+	p, err := profile.Parse("draft-snmpv2-mib.yaml", []byte(fillTODOs(d)))
+	if err != nil {
+		t.Fatalf("%v\n%s", err, d)
+	}
+	if len(p.Metrics) != 2 || p.Metrics[0].OID != "1.3.6.1.2.1.1.3.0" || p.Metrics[0].Unit != "seconds" {
+		t.Errorf("metrics = %+v", p.Metrics)
+	}
+}
+
+func TestProfileNewHintAndUnits(t *testing.T) {
+	d := runNew(t, "--mib", "TEST-MIB", "--mibs-dir", "../../internal/mib/testdata")
+	for _, want := range []string{"suggest scale: 0.1", "oid: 1.3.6.1.4.1.99999.1.1.0", "unit: celsius", "type: status", "2: \"on\"",
+		"index_label: { column: testName, as: component }"} {
+		if !strings.Contains(d, want) {
+			t.Errorf("missing %q in:\n%s", want, d)
+		}
+	}
+	if _, err := profile.Parse("draft-test-mib.yaml", []byte(fillTODOs(d))); err != nil {
+		t.Fatalf("%v\n%s", err, d)
+	}
+}
+
+func TestProfileNewErrors(t *testing.T) {
+	var out bytes.Buffer
+	for _, args := range [][]string{
+		{},
+		{"--mib", "NO-SUCH-MIB", "--mibs-dir", "../../internal/mib/testdata"},
+		{"--mib", "TEST-MIB", "--table", "nope", "--mibs-dir", "../../internal/mib/testdata"},
+		{"--mib", "TEST-MIB", "--objects", "testTable", "--mibs-dir", "../../internal/mib/testdata"},
+	} {
+		if err := profileNew(context.Background(), args, &out); err == nil {
+			t.Errorf("%v: expected an error", args)
+		}
+	}
+}
+
+const recordSample = `1.3.6.1.2.1.1.1.0|4|Test device
+1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.8072.3.2.10
+1.3.6.1.2.1.1.3.0|67|123456
+1.3.6.1.2.1.1.4.0|4|admin@example.com
+1.3.6.1.2.1.1.5.0|4|secret-comm
+1.3.6.1.2.1.1.6.0|4|Rack 12, Building 3
+1.3.6.1.2.1.2.1.0|2|1
+1.3.6.1.2.1.2.2.1.1.1|2|1
+1.3.6.1.2.1.2.2.1.10.1|65|42
+1.3.6.1.2.1.31.1.1.1.6.1|70|18446744073709551615
+1.3.6.1.4.1.9999.1.1.0|4|secret-comm
+1.3.6.1.4.1.9999.1.2.0|4x|00ff
+1.3.6.1.4.1.9999.1.3.0|64|10.1.2.3
+`
+
+func TestProfileRecordRedacts(t *testing.T) {
+	rec, err := snmprec.Parse(strings.NewReader(recordSample))
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	ag, err := simagent.Start(ctx, simagent.Config{Recording: rec, Community: "secret-comm"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = ag.Close() }() // test cleanup
+	addr := ag.Addr()
+	for _, ver := range []string{"2c", "1"} {
+		var out bytes.Buffer
+		err := profileRecordTo(ctx, &out, addr, ver)
+		if err != nil {
+			t.Fatalf("v%s: %v", ver, err)
+		}
+		s := out.String()
+		for _, secret := range []string{"secret-comm", "admin@example.com", "Rack 12"} {
+			if strings.Contains(s, secret) {
+				t.Errorf("v%s: recording leaks %q:\n%s", ver, secret, s)
+			}
+		}
+		if n := strings.Count(s, "REDACTED"); n != 4 {
+			t.Errorf("v%s: want 4 REDACTED values, got %d:\n%s", ver, n, s)
+		}
+		got, err := snmprec.Parse(strings.NewReader(s))
+		if err != nil {
+			t.Fatalf("v%s: output does not parse: %v", ver, err)
+		}
+		want := rec.Len()
+		if ver == "1" {
+			want-- // Counter64 does not exist in SNMPv1
+		}
+		if got.Len() != want {
+			t.Errorf("v%s: %d records, want %d\n%s", ver, got.Len(), want, s)
+		}
+	}
+}
+
+func profileRecordTo(ctx context.Context, out *bytes.Buffer, addr netip.AddrPort, ver string) error {
+	return profileRecord(ctx, []string{addr.Addr().String(), "--port", itoa(int(addr.Port())), "--community", "secret-comm",
+		"--version", ver, "--timeout", "1s", "--subtree", "1.3.6.1.2.1", "--subtree", "1.3.6.1.4.1"}, out)
+}
+
+func itoa(n int) string { return strconv.Itoa(n) }
+
+func TestProfileRecordArgs(t *testing.T) {
+	var out bytes.Buffer
+	for _, args := range [][]string{{}, {"10.0.0.1"}, {"nonsense", "--community", "x"}, {"10.0.0.1", "--version", "3"}} {
+		if err := profileRecord(context.Background(), args, &out); err == nil {
+			t.Errorf("%v: expected an error", args)
+		}
+	}
+}
+
+func TestProfileTestArgs(t *testing.T) {
+	var out bytes.Buffer
+	for _, args := range [][]string{{}, {"10.0.0.1"}, {"nonsense", "--community", "x"}, {"10.0.0.1", "--version", "v3"}, {"10.0.0.1", "--version", "v9", "--community", "x"}} {
+		if err := profileTest(context.Background(), args, &out); err == nil {
+			t.Errorf("%v: expected an error", args)
+		}
+	}
+}
+
+func TestProfileTestPrintsMetrics(t *testing.T) {
+	rec, err := snmprec.Load("../../testdata/snmprec/juniper-junos.snmprec")
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, err := simagent.Start(context.Background(), simagent.Config{Recording: rec, Community: "lab"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = a.Close() }() // test cleanup
+	var out bytes.Buffer
+	err = profileTest(context.Background(), []string{a.Addr().Addr().String(), "--port", itoa(int(a.Addr().Port())), "--community", "lab"}, &out)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, out.String())
+	}
+	for _, want := range []string{"profile      juniper-junos", "ifHCInOctets", "jnxOperatingCPU", "PDUs per cycle", "0 type mismatches"} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("output lacks %q:\n%s", want, out.String())
+		}
+	}
+}
+
+// The LibreNMS importer is a separate offline tool (DEVIATIONS #9): nmon must not link it,
+// and neither may nmon pull in the code that reads LibreNMS definitions.
+func TestNmonDoesNotImportLibreNMS(t *testing.T) {
+	out, err := exec.Command("go", "list", "-deps", ".").CombinedOutput()
+	if err != nil {
+		t.Skipf("go list unavailable: %v\n%s", err, out)
+	}
+	for _, pkg := range strings.Fields(string(out)) {
+		if strings.Contains(pkg, "internal/librenms") || strings.Contains(pkg, "nmon-librenms-import") {
+			t.Errorf("cmd/nmon depends on %s", pkg)
+		}
+	}
+}
+
+// Dev and test machinery stays out of the product binary: nmon-devtools and tests own the simulator,
+// the fixture generator and the fixtures. `nmon profile record` needs only internal/snmprec.
+func TestNmonDoesNotLinkDevMachinery(t *testing.T) {
+	out, err := exec.Command("go", "list", "-deps", ".").CombinedOutput()
+	if err != nil {
+		t.Skipf("go list unavailable: %v\n%s", err, out)
+	}
+	banned := []string{"internal/simagent", "internal/fixturegen", "internal/fixtures", "internal/librenms", "internal/testutil"}
+	for _, pkg := range strings.Fields(string(out)) {
+		for _, b := range banned {
+			if strings.HasSuffix(pkg, "/"+b) {
+				t.Errorf("cmd/nmon depends on %s", pkg)
+			}
+		}
+	}
+}

cmd/nmon/restore.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/v10networks/nmon/internal/app"
+	"github.com/v10networks/nmon/internal/backup"
+	"github.com/v10networks/nmon/internal/config"
+	"github.com/v10networks/nmon/internal/vmsup"
+)
+
+func init() {
+	register("restore", "restore a backup into a clean data dir (nmon stopped)", restoreCmd)
+}
+
+// restoreCmd unpacks a backup into an empty data directory and verifies it before it reports success. It
+// runs with nmon stopped, because it starts the restored metric stores itself to compare their series.
+func restoreCmd(ctx context.Context, args []string) error {
+	var f config.Flags
+	fs := newFlagSet("restore", "restore FILE --data-dir CLEAN_DIR [--secret-key-file PATH]")
+	fs.StringVar(&f.DataDir, "data-dir", "", "the clean (absent or empty) data folder to restore into (required)")
+	fs.StringVar(&f.SecretKeyFile, "secret-key-file", "",
+		"where to write the key file if the backup contains one (default <data-dir>/secret.key; nmon serve wants it outside the data dir)")
+	file, rest, err := leadingArg(args)
+	if err != nil {
+		return err
+	}
+	if err := fs.Parse(rest); err != nil {
+		return flagErr(err)
+	}
+	if file == "" && fs.NArg() == 1 {
+		file = fs.Arg(0)
+	}
+	if file == "" || f.DataDir == "" || fs.NArg() > 1 {
+		return errors.New("usage: nmon restore FILE --data-dir CLEAN_DIR")
+	}
+	res, err := backup.Restore(ctx, backup.RestoreOptions{
+		File: file, DataDir: f.DataDir, KeyFile: f.SecretKeyFile, Log: os.Stdout,
+		Verifier: backup.NewVMVerifier(vmsup.Options{DataDir: f.DataDir}),
+	})
+	if err != nil {
+		return err
+	}
+	fmt.Printf("restore: done and verified: %d inventory rows and %d series matched the manifest\n", res.RowsVerified, res.SeriesVerified)
+	if res.KeyRestoredTo != "" {
+		fmt.Printf("restore: the encryption key was restored to %s\n", res.KeyRestoredTo)
+	} else {
+		fmt.Printf("restore: this backup has NO key file; start nmon with --secret-key-file pointing at the key that sealed it "+
+			"(default %s)\n", app.DefaultSecretKeyFile)
+	}
+	return nil
+}

cmd/nmon/rollup.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/v10networks/nmon/internal/cliout"
+	"github.com/v10networks/nmon/internal/rollup"
+)
+
+func init() {
+	register("rollup", "repair rollup windows (asks a running nmon)", rollupCmd)
+}
+
+const rollupUsage = "usage: nmon rollup repair <from> <to> [--api URL] [--token TOKEN]\n\n" +
+	"Re-runs every rollup window overlapping [from, to) on the running nmon, writing a new version of\n" +
+	"each. The range must be inside hot retention (default 30 days). Times are RFC 3339 (2026-10-20T08:00:00Z)\n" +
+	"or a UTC date (2026-10-20); to is exclusive."
+
+func rollupCmd(ctx context.Context, args []string) error {
+	if len(args) == 0 || args[0] != "repair" {
+		return fmt.Errorf("%s", rollupUsage)
+	}
+	fs := profileFlagSet("rollup repair", "rollup repair <from> <to> [--api URL]")
+	var api, token string
+	var timeout time.Duration
+	fs.StringVar(&api, "api", "http://[::1]:8443", "base URL of the running nmon")
+	fs.StringVar(&token, "token", os.Getenv("NMON_TOKEN"), "admin API token (default $NMON_TOKEN)")
+	fs.DurationVar(&timeout, "timeout", 30*time.Minute, "how long to wait for the repair")
+	pos, err := parseInterspersed(fs, args[1:])
+	if err != nil {
+		return flagErr(err)
+	}
+	if len(pos) != 2 {
+		return fmt.Errorf("%s", rollupUsage)
+	}
+	from, err := parseRepairTime(pos[0])
+	if err != nil {
+		return err
+	}
+	to, err := parseRepairTime(pos[1])
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	rep, err := requestRepair(ctx, api, token, rollup.RepairRequest{From: from, To: to})
+	if err != nil {
+		return err
+	}
+	w := cliout.New(os.Stdout)
+	w.Printf("repaired %s to %s: %d five-minute windows, %d hourly windows rewritten\n",
+		rep.From.Format(time.RFC3339), rep.To.Format(time.RFC3339), rep.FiveMinutes, rep.Hours)
+	return w.Err()
+}
+
+func parseRepairTime(s string) (time.Time, error) {
+	for _, layout := range []string{time.RFC3339, "2006-01-02T15:04", "2006-01-02"} {
+		if t, err := time.ParseInLocation(layout, s, time.UTC); err == nil {
+			return t.UTC(), nil
+		}
+	}
+	return time.Time{}, fmt.Errorf("cannot parse %q as RFC 3339 or YYYY-MM-DD", s)
+}
+
+// requestRepair calls POST /api/v1/rollup/repair, which internal/app mounts on
+// rollup.Job.Repair.
+func requestRepair(ctx context.Context, api, token string, rr rollup.RepairRequest) (*rollup.RepairReport, error) {
+	body, err := json.Marshal(rr)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(api, "/")+"/api/v1/rollup/repair", bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot reach nmon at %s (is it running? use --api): %w", api, err)
+	}
+	defer func() { _ = resp.Body.Close() }() // body fully read below; a close error changes nothing
+	out, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if resp.StatusCode != http.StatusOK {
+		var p struct{ Title, Detail string }
+		_ = json.Unmarshal(out, &p)
+		msg := p.Detail
+		if msg == "" {
+			msg = p.Title
+		}
+		return nil, fmt.Errorf("repair refused: %d %s", resp.StatusCode, msg)
+	}
+	var rep rollup.RepairReport
+	if err := json.Unmarshal(out, &rep); err != nil {
+		return nil, err
+	}
+	return &rep, nil
+}

cmd/nmon/rollup_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/rollup"
+)
+
+func TestRollupRepairSendsRangeAndTokenToRunningNmon(t *testing.T) {
+	var got rollup.RepairRequest
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v1/rollup/repair" || r.Header.Get("Authorization") != "Bearer tok" {
+			http.Error(w, "bad request", http.StatusBadRequest)
+			return
+		}
+		_ = json.NewDecoder(r.Body).Decode(&got)
+		_ = json.NewEncoder(w).Encode(rollup.RepairReport{From: got.From, To: got.To, FiveMinutes: 12, Hours: 1})
+	}))
+	defer srv.Close()
+	from, to := time.Date(2026, 10, 20, 8, 0, 0, 0, time.UTC), time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)
+	rep, err := requestRepair(context.Background(), srv.URL, "tok", rollup.RepairRequest{From: from, To: to})
+	if err != nil || rep.FiveMinutes != 12 || !got.From.Equal(from) || !got.To.Equal(to) {
+		t.Fatalf("rep=%+v got=%+v err=%v", rep, got, err)
+	}
+	if _, err := requestRepair(context.Background(), srv.URL, "wrong", rollup.RepairRequest{}); err == nil {
+		t.Error("a refused repair must be an error")
+	}
+}
+
+func TestParseRepairTimeAcceptsDatesAndRFC3339(t *testing.T) {
+	for in, want := range map[string]time.Time{
+		"2026-10-20":           time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC),
+		"2026-10-20T08:30":     time.Date(2026, 10, 20, 8, 30, 0, 0, time.UTC),
+		"2026-10-20T08:00:00Z": time.Date(2026, 10, 20, 8, 0, 0, 0, time.UTC),
+	} {
+		if got, err := parseRepairTime(in); err != nil || !got.Equal(want) {
+			t.Errorf("%s: %v %v", in, got, err)
+		}
+	}
+	if _, err := parseRepairTime("yesterday"); err == nil {
+		t.Error("want error")
+	}
+}

cmd/nmon/serve.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"log/slog"
 	"os"
 
 	"github.com/v10networks/nmon/internal/app"
+	"github.com/v10networks/nmon/internal/cliout"
 	"github.com/v10networks/nmon/internal/config"
 )
 
@@ -23,7 +25,9 @@ var hidden = map[string]bool{"poll-interval": true, "snmp-port": true}
 func newFlagSet(name, usage string) *flag.FlagSet {
 	fs := flag.NewFlagSet(name, flag.ContinueOnError)
 	fs.Usage = func() {
-		fmt.Fprintf(fs.Output(), "usage: nmon %s\n\nflags:\n", usage)
+		// Usage text is best-effort; a failed write to the terminal has nowhere to go.
+		out := cliout.New(fs.Output())
+		out.Printf("usage: nmon %s\n\nflags:\n", usage)
 		fs.VisitAll(func(f *flag.Flag) {
 			if hidden[f.Name] {
 				return
@@ -32,7 +36,7 @@ func newFlagSet(name, usage string) *flag.FlagSet {
 			if f.DefValue != "" && f.DefValue != "false" && f.DefValue != "0" {
 				def = fmt.Sprintf(" (default %s)", f.DefValue)
 			}
-			fmt.Fprintf(fs.Output(), "  --%s\n        %s%s\n", f.Name, f.Usage, def)
+			out.Printf("  --%s\n        %s%s\n", f.Name, f.Usage, def)
 		})
 	}
 	return fs
@@ -47,13 +51,31 @@ func storeFlags(fs *flag.FlagSet, f *config.Flags) {
 
 func serve(ctx context.Context, args []string) error {
 	var f config.Flags
-	fs := newFlagSet("serve", "serve --data-dir DIR --insecure-http [flags]")
+	fs := newFlagSet("serve", "serve --data-dir DIR (--tls-cert F --tls-key F | --acme-host H | --insecure-http) [flags]")
 	storeFlags(fs, &f)
-	fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile, "SNMP credential key file; created on first run; must be outside --data-dir")
+	fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile,
+		"SNMP credential key file; created on first run; must be outside --data-dir")
 	fs.StringVar(&f.Listen, "listen", app.DefaultListen, "address for the API")
-	fs.BoolVar(&f.InsecureHTTP, "insecure-http", false, "serve plain HTTP with no authentication (required until TLS and auth arrive in M6)")
+	fs.BoolVar(&f.InsecureHTTP, "insecure-http", false,
+		"development only: serve plain HTTP (insecure cookies); with no auth.oidc or auth.break_glass_file configured, "+
+			"authentication is off and every request is an admin")
+	fs.StringVar(&f.TLSCert, "tls-cert", "", "PEM certificate chain for HTTPS (reloaded on SIGHUP); needs --tls-key")
+	fs.StringVar(&f.TLSKey, "tls-key", "", "PEM private key for --tls-cert")
+	fs.Func("acme-host", "obtain a Let's Encrypt certificate for this host name (repeatable); needs port 80 reachable", func(v string) error {
+		f.ACMEHosts = append(f.ACMEHosts, v)
+		return nil
+	})
+	fs.StringVar(&f.ACMEEmail, "acme-email", "", "contact address for the ACME account")
+	fs.StringVar(&f.ACMEListen, "acme-listen", app.DefaultACMEListen, "address of the ACME http-01 challenge listener")
 	fs.StringVar(&f.VMHotURL, "vm-hot-url", "", "use this VictoriaMetrics instance as the hot store instead of the embedded one")
 	fs.StringVar(&f.VMLongURL, "vm-long-url", "", "use this VictoriaMetrics instance as the long store instead of the embedded one")
+	fs.StringVar(&f.TrapListen, "trap-listen", app.DefaultTrapListen,
+		"UDP address for SNMP traps; empty disables the receiver "+
+			"(port 162 needs CAP_NET_BIND_SERVICE; without it traps are off and polling continues)")
+	fs.StringVar(&f.GraphiteListen, "graphite-listen", app.DefaultGraphiteListen,
+		"TCP address for Graphite plaintext push (first line \"token <secret>\"); empty disables the listener")
+	fs.StringVar(&f.ICMP, "icmp", config.ICMPAuto,
+		"ICMP probing: auto, or off for hosts where ICMP is not permitted (SNMP alone then decides if a device is down)")
 	fs.IntVar(&f.MaxInflight, "max-inflight", 512, "most SNMP polls in flight at once")
 	fs.DurationVar(&f.PollInterval, "poll-interval", 0, "dev: SNMP poll interval (default 60s)")
 	fs.IntVar(&f.SNMPPort, "snmp-port", 0, "dev: UDP port of every device's agent (default 161)")
@@ -70,7 +92,7 @@ func serve(ctx context.Context, args []string) error {
 
 // flagErr turns flag.ErrHelp into a clean exit.
 func flagErr(err error) error {
-	if err == flag.ErrHelp {
+	if errors.Is(err, flag.ErrHelp) {
 		os.Exit(0)
 	}
 	return err

docs/DEVIATIONS.md

@@ -14,3 +14,31 @@ conscious, reviewed exception.
 | 7 | `import-librenms` reads `includes/definitions/*.yaml` and `includes/definitions/discovery/*.yaml` | Reads `resources/definitions/os_detection/*.yaml` and `resources/definitions/os_discovery/*.yaml`, with the old paths as a fallback. | LibreNMS moved them; the spec's paths no longer exist upstream. | M2 |
 | 8 | Real recordings replace synthetic fixtures when available | LibreNMS's `tests/snmpsim/*.snmprec` (2,054 real device recordings) are used by optional tests when `NMON_LIBRENMS_DIR` points at a checkout. They are never copied into this repo. | Real-device coverage without vendoring GPL test data into an Apache-2.0 repo. | M2 |
 | 9 | `nmon profile import-librenms <repo-path>` is an nmon subcommand | A separate binary, `nmon-librenms-import` (`cmd/nmon-librenms-import`), run offline before nmon. It writes draft profiles that the operator reviews and copies into `<data-dir>/profiles/`. `nmon` neither links nor exposes it. | User decision (2026-10-22). It keeps the importer out of the product binary and makes "the converter never runs at serve time" structural. | M2 |
+| 10 | `scripts/openapi-diff` fails on an incompatible change (M3 milestone) | The differ is the Go package `internal/openapidiff` (unit-tested, run with `go run ./internal/openapidiff/cmd/openapidiff`), driven by `scripts/openapi-check.sh`, which `make gate-m3` and the CI `openapi` job call. | A Go package is testable and linted like the rest of the code; the script only fetches the spec at the last `m*`/`v*` tag. No new dependency. | M3 |
+| 11 | Built assets are embedded from `web/dist` into `internal/webui` | `make web` copies `web/dist` to `internal/webui/dist` (git-ignored, `.gitkeep` tracked); a committed `internal/webui/placeholder/index.html` ("UI not built") is served when `dist` has no `index.html`. | A tracked `dist/index.html` would be overwritten by every build and dirty the tree, and a built UI must never be committed. `go build` still works without node. | M3 |
+| 12 | (not specified) | Developer tools (`backfill`, `seed-demo`) are a separate executable, `cmd/nmon-devtools` (`make devtools` builds `nmon-devtools`), never shipped. It talks to a running nmon over its HTTP API and to the hot store via `--hot-url`. `nmon` has no `devtools` command; depguard and a `go list -deps` test keep `cmd/nmon` from linking `internal/simagent`, `internal/fixturegen`, `internal/fixtures` or `internal/testutil`; `nmon profile record` uses only the `.snmprec` format in `internal/snmprec`. | The gate needs 30 days of history without waiting 30 days, and live screenshots need a simulated network. See `docs/dev.md`. | M3 |
+| 13 | (not specified) `/docs` API reference | `/docs` serves a vendored, pinned swagger-ui-dist 5.17.14 (Apache-2.0, about 1.6 MB) through `embed.FS`. | huma's built-in renderers load their assets from a CDN, which an offline monitor cannot rely on. | M3 |
+| 14 | M2 defined provisional `poller/health` and `identity-reviews` responses | Both were reshaped in M3 to the reviewed UI contract, before any `openapi.json` existed, so the route freeze starts at `m3`, the first tag with a checked-in spec. | The M2 shapes had no consumer. `scripts/openapi-check.sh` passes against `m2` with a note and diffs for real from `m3`. | M3 |
+| 15 | A single-signal failure (ICMP only or SNMP only) is a warning | The warning is held as pending for 5 min before it notifies, and is closed silently if the device goes down meanwhile. | Otherwise a real outage whose second signal fails seconds later sends a spurious warning first. | M4 |
+| 16 | An unsupported probe is unknown, never failed | When ICMP is unsupported (host can't open unprivileged ICMP, or per device), SNMP alone decides down, as with `icmp_blocked`, and a one-time `icmp_unsupported` device warning is raised. | Otherwise nmon could never declare a device down on such a host, and nobody would notice. | M4 |
+| 17 | M6 gate: "the 2,000-device loadtest cycle finishes under 40 s"; the loadtest acceptance lines (200k interfaces in 40 s, memory stable over 30 min, peak RSS/disk/ingest/latency report, backup during load) | **Skipped** (approved by the owner, 2026-10-22: "skip the device load test… put this in prod asap"). The M6 gate checks only that a viewer gets 403 on PUT. Backup/restore is verified on a normal-size dataset instead of under load. Scale is unproven beyond the fixture and gate sizes (24 devices). | Time to production. | M6 |
+| 18 | Push token format not specified | The secret is `nmp_<id>_<base32 of 160 random bits>`; the row stores a salted SHA-256 (`<salt>$<hash>`). | The id selects the row, so authentication is one primary-key lookup and never scans or compares other tokens. | M5 |
+| 19 | `rollup_windows` status is `done` or `partial` | A third status, `pending`, is written before the long-store write begins. | A crash between the write and the commit still tells the next run which version number is taken, so it writes `v+1`. | M5 |
+| 20 | `v` is a version number on rollup series | `v` is on `nmon_if_rate_bps_5m`, `_avg_1h`, `_max_1h` and the gauge series, encoded `YYYYMM*1000+n` (UTC month of the window, then the version). | Numeric, so MetricsQL can compare versions; the month prefix lets retention drop whole months of 5 m series. Readers use `internal/longq.LatestVersion`. | M5 |
+| 21 | Gauge rollups: "a 5 m average, plus a 1 h avg, max and min" | Series are named `<name>_avg_5m`, `<name>_avg_1h`, `<name>_max_1h`, `<name>_min_1h`, with the source labels plus `v`. | The suffix keeps the source name searchable and cannot collide with a polled metric (such names are not rolled up). | M5 |
+| 22 | 5 m series are deleted past `retention.fiveminute` | Whole calendar months are deleted with `delete_series` once they ended before `now - retention.fiveminute`, so 5 m data lives 4 to 5 months. | Open-source VictoriaMetrics has one retention per process and no time-ranged delete; the `v` prefix makes each 5 m series one month. | M5 |
+| 23 | Rollup writes go through the write queue | Rollup long writes bypass the queue: a synchronous, acknowledged import, then `force_flush`, then a read-back count. | The queue is fire-and-forget (it can drop after 60 s); a window may be recorded done only once the store has it. | M5 |
+| 24 | Unreliable (32-bit) interfaces are excluded from p95 and group sums | They are also never rolled up, and a graph of one returns no series. | A rate from a wrapping counter is wrong whichever tier stores it. | M5 |
+| 25 | `nmon_group_p95_bps{group, dir, month, members_hash}` | Group (and interface) p95 records also carry a `coverage` label. | A figure below 90% coverage must never be shown without it. | M5 |
+| 26 | Check samples labelled by device | `nmon_check_up`, `nmon_check_latency_seconds` and `nmon_tls_expiry_seconds` also carry `check` (the check id). | A device can have several checks. | M5 |
+| 27 | Proxmox nodes are devices | A node is the existing SNMP device with that name (or sysName), else a new `check_only` device; its health is the status the API reports. Guests are `vm`/`container` devices keyed by `pve:<cluster>/<vmid>`. | `cluster/resources` has no addresses, so nothing but the name can match a node; a missing SNMP device must not orphan its guests. | M5 |
+| 28 | Pushed series carry only `device` | Pushed series also carry `push="1"` (senders cannot set it). | The device page lists what a push sender wrote without guessing from metric names. | M5 |
+| 29 | Ingest caps, late p95 months and a stalled rollup "raise an alert" | They are rows of the alerts table with rules `ingest_<kind>`, `p95_late`, `rollup_stalled` and no device, kept open until cleared (ingest alarms close 20 min after the last trip). | They are not metric rules and have no device; the metric-rule evaluator leaves these rule names alone. | M5 |
+| 30 | Proxmox cluster config could carry the token secret | Only `secret_name` exists; the secret is stored encrypted via `PUT /api/v1/secrets/{name}` and cannot be read back. | `config.yaml` is plain text. | M5 |
+| 31 | HTTPS by default; plain HTTP only with `--insecure-http` (Auth: OIDC only) | With `--insecure-http` **and** neither `auth.oidc` nor `auth.break_glass_file` configured, nmon runs in **dev-open mode**: every request is the admin principal `dev`, a loud warning is logged at start and the UI shows "Authentication disabled — development mode". Any start over TLS with no sign-in method configured refuses to start. If either sign-in method is configured, `--insecure-http` only means plain HTTP and cookies without `Secure`; authentication stays on. | The M1-M5 gates (and local development) run `nmon serve --insecure-http` with no IdP; M6 must not break them. The refusal for TLS-without-auth keeps the unsafe mode from ever being a production start. | M6 |
+| 32 | Settings (retention, alert thresholds, IdP, receivers, outbound allow-list) are edited in the UI and reloaded | `PUT /api/v1/settings` and SIGHUP reload in place: alert receivers, the outbound allow-list, trusted proxies, the nightly backup schedule, disk thresholds and the retired-device purge period. Alert rules and their interval, retention, timezone and sites, Proxmox, checks, `auth.oidc`, break-glass, session lifetimes, listen address, TLS and the data dir are saved but reported as `restart_required`. An invalid document is a 422 and changes nothing; the file is rewritten from the validated structure, so comments in a hand-edited `config.yaml` are lost by a UI save. | Those parts are built once at start (stores, evaluators, the OIDC client); rebuilding them live is a larger change than the risk of a restart warrants for v1. | M6 |
+| 33 | `nmon backup <file>` takes an online snapshot | `nmon backup` is a client of the running nmon: it calls `POST /api/v1/admin/backup` (admin token), nmon runs the backup in a scratch file under `<data-dir>/backups/.downloads/`, streams it and deletes it, and the client saves it atomically (temp file + rename). The key-included line is printed from the archive's own manifest. `nmon restore` runs offline and starts the restored stores for a moment to verify series. | Only the running process may touch the live stores and the database. | M6 |
+| 34 | (not specified) | Beyond the spec, additive to the API/CLI: `GET /api/v1/me`, `GET /api/v1/auth/options` (public), `GET /api/v1/audit` (admin), `POST /api/v1/admin/backup`, `POST /api/v1/admin/stores/{name}/restart`, `POST /api/v1/devices/{slug}/purge`, `POST /api/v1/devices/import`, `GET/PUT /api/v1/settings`; `nmon secret set NAME` (stores the OIDC client secret before the first start; it also creates the key on a brand-new install); API tokens (viewer, admin) share `/api/v1/tokens` with push tokens and have ids like `a7` because the two kinds live in different tables. | OIDC needs its client secret at start, while the API that stores secrets needs a way to sign in: the CLI breaks the cycle. The audit list makes "every use is logged" checkable. | M6 |
+| 35 | Port descriptions imported from CSV | Interfaces have a new operator-owned `description` column (migration 8). The CSV import and the API write it; discovery rewrites `ifalias` from the device on every run and never touches `description`. Interface search `q` matches ifname, ifalias and description. | A description typed into `ifalias` would be overwritten at the next discovery. | M6 |
+| 36 | OIDC login with a browser flow; break-glass as a local login | **No login UI** (owner decision). Every route except `/auth/*`, `/healthz` and push ingest needs a credential, the SPA's files included: an unauthenticated page load is a 302 to `/auth/login?return=<url>` (the OIDC flow; the return target must be a same-site path), an API call without credentials is a 401 problem+json. Break-glass is `GET /auth/breakglass` with HTTP Basic, so the browser's own prompt appears; success sets a session and redirects to `return` or `/` (`POST /auth/breakglass` with JSON stays for scripts). When no IdP is configured the redirect goes to the break-glass prompt. Logout is `POST /auth/logout`, then `/auth/signedout` (static HTML). `/docs` and `/openapi*` need the viewer role. A missing OIDC client secret at start leaves OIDC off with a warning. | The IdP is the login page; a second one would be a second thing to secure. | M6 |
+| 37 | Disk guard: "ingest throttled below the reserve" | Throttle answers push ingest with 429 + Retry-After below the 5% reserve; the poller keeps writing (its volume is bounded by the inventory). Above 90% used, new series are dropped on the nmon side (poller, push and checks) while known series continue; disk alarms are re-raised each minute while the condition holds and resolve themselves 3 minutes after it clears. | VictoriaMetrics series limits are start-up flags; the guard is nmon-side admission. | M6 |

docs/MAINTAINABILITY.md

@@ -0,0 +1,47 @@
+# Maintainability standard
+
+**This is a hard rule** (owner decision, 2026-10-22). Code that breaks it does not merge, and a milestone with findings is not tagged. The machine enforces it: `make lint` runs golangci-lint with `.golangci.yml` (Go) and ESLint (web). Every `scripts/gate-m*.sh` runs `make lint` first, and so does CI.
+
+## Limits (production code)
+
+| What | Limit | Enforced by |
+|---|---|---|
+| Function length | 80 lines, 50 statements | `funlen` |
+| Cyclomatic complexity | 15 per function | `cyclop` |
+| Cognitive complexity | 20 per function | `gocognit` |
+| Nesting | `if` depth 4 | `nestif` |
+| Duplicated blocks | 150 tokens | `dupl` |
+| Line length | 140 characters | `lll` |
+| File length | 600 lines (split by concern before that) | `scripts/lint-filesize.sh` |
+| Unused code, dead fields | none | `unused` |
+| Deprecated APIs | none | `staticcheck` |
+| Unchecked errors | none | `errcheck`, `errorlint` |
+| Magic HTTP status numbers | none (use `http.Status*`) | `usestdlibvars` |
+
+Tests (`_test.go`) are exempt from the length and complexity limits, because table-driven tests are long by nature. They must still pass every other linter.
+
+## Architecture rules (enforced by `depguard` and `forbidigo`)
+
+- **SQL and the ORM live only in `internal/store`.** No other package imports `github.com/uptrace/bun` or `database/sql`.
+- **Layering:** `cmd → app → {api, poller, discovery, traps, alert, notify, …} → {store, snmp, profile, metrics, clock, …}`.
+  - Only `cmd/*` imports `internal/app`.
+  - Only `internal/app` (and tests) imports `internal/api`.
+  - Collection paths do not import each other: `poller`, `discovery`, `traps`, `ingest`, `checks`, `rollup` and `alert` share code through a lower package such as `internal/snmpx`, not through each other.
+  - `cmd/nmon` never imports `internal/librenms` (the importer is a separate tool).
+- **Time comes from `internal/clock`.** `time.Now`, `time.Sleep`, `time.After`, `time.Tick` and `time.NewTicker` are forbidden outside `internal/clock`, `cmd/` and tests.
+- **No global mutable state** except the `selfmon.Default` registry and package-level caches that are documented as such.
+
+## Habits the linters can't check (reviewers enforce these)
+
+- One concern per file. Name files after what they hold (`matcher.go`, not `util.go`). No `util`, `helpers` or `common` packages.
+- Split a long function into named steps that read like the spec (`collect → match → apply → audit`), not into `doPart1/doPart2`.
+- Errors are wrapped with context (`fmt.Errorf("discovery %s: %w", slug, err)`) and handled once.
+- Every exported identifier has a doc comment that says *why* or *what contract*, not a restatement of the name.
+- Comments explain decisions, not mechanics. A decision that departs from the spec goes in `docs/DEVIATIONS.md`.
+- Tests are named for the behaviour they lock in (`TestStackSwapOpensOneReviewAndNeverMerges`).
+- `//nolint` is allowed only with the linter named and a reason (`//nolint:funlen // generated table`). `nolintlint` enforces this.
+- New dependencies need a line in `docs/DEVIATIONS.md` row 3 saying why the standard library isn't enough.
+
+## Web (`web/`)
+
+ESLint with typescript-eslint (strict), `max-lines-per-function` 80, `complexity` 15, `max-depth` 4, `max-lines` 400 per file, no `any`, react-hooks rules. `TrafficGraph.tsx` is the only file that imports uPlot. API types are only generated, never hand-written.

docs/dev.md

@@ -0,0 +1,59 @@
+# Developer notes
+
+## Building the UI into the binary
+
+`make build` runs `make web` first: `npm run build` in `web/`, then `web/dist` is copied to
+`internal/webui/dist`, which `//go:embed` packs into the binary. `internal/webui/dist` is git-ignored
+(only `.gitkeep` is tracked), so a build never dirties the tree. A plain `go build` without `make web`
+embeds a one-line "UI not built" page from `internal/webui/placeholder/`.
+
+nmon serves the UI at `/`. Unknown paths fall back to `index.html` for the client-side router, except
+`/api/`, `/docs`, `/openapi*`, `/metrics` and `/schemas`, which belong to the API. Files under `/assets/`
+are content-hashed and sent `immutable`; `index.html` is `no-cache`; text assets are served gzipped.
+
+## OpenAPI compatibility
+
+`scripts/openapi-check.sh` compares the current spec (`nmon openapi`, else `openapi.json`) with
+`openapi.json` at the latest `m*`/`v*` tag and fails on a removed route or method, a removed response
+field, a changed field type, a new required parameter or request field, or a removed enum value. A tag
+without `openapi.json` passes with a note. `OPENAPI_NEW=file.json` compares that file instead and
+`OPENAPI_BASE_TAG=m3` picks the base. The comparison is `internal/openapidiff`.
+
+## `nmon-devtools` (separate dev binary)
+
+Developer helpers in their own executable, `make devtools` -> `nmon-devtools`; the shipped `nmon` does not contain them and never links the simulator. They need a running `nmon serve --insecure-http`.
+
+**backfill** writes synthetic history straight into the hot store so graphs have 30 days of data without
+waiting 30 days:
+
+    nmon-devtools backfill --hot-url http://127.0.0.1:PORT --device SLUG --iface ULID [--iface ULID...] \
+        [--days 31] [--step 60s] [--pattern diurnal|flat] [--mean-mbps N]
+
+Per interface it writes `ifHCInOctets`, `ifHCOutOctets` (monotonic, diurnal rate with noise and bursts) and
+`ifOperStatus`, ending one step before the first live sample. When live samples exist the history is shifted
+so its last value joins the live counter with no reset (this needs a live counter larger than the history,
+about 3e13 bytes for 31 days at 100 Mbit/s; otherwise a warning says the join will look like a reset).
+The hot URL is in the serve log (`metric stores up hot=...`). Samples older than hot retention are dropped
+by the store.
+
+**seed-demo** builds a small simulated network and back-fills it, for the gate and for screenshots:
+
+    nmon serve --data-dir D --insecure-http --poll-interval 10s  ...
+    nmon-devtools seed-demo --api http://127.0.0.1:8443 --hot-url http://127.0.0.1:PORT \
+        [--env-out env] [--counters-rec testdata/snmprec/counters.snmprec]
+
+It starts the simulated SNMP agents in-process on `[::1]`, each on its own port (passed to the API as `snmp.port`), adds the devices through the API (a 48-port switch, access switches, routers, a firewall, a UPS, the counters fixture, a v1 device with 32-bit counters, and one whose agent is stopped after seeding so it reads as down), waits for live polls, back-fills 31 days, writes `NMON_DEVICE`, `NMON_DEVICE_48`, `NMON_DEVICE_32BIT` and `NMON_DEVICE_DOWN` to `--env-out`, and keeps the agents answering until interrupted. The simulated 64-bit counters start at 1e14 so history fits below them.
+
+**oidc-mock** runs `internal/auth/oidctest` as a standalone OpenID Connect issuer for the M6 gate and for screenshots:
+
+    nmon-devtools oidc-mock --issuer-file F --port-file P [--client-id nmon --client-secret s3cret]
+    curl 'http://[::1]:PORT/user?sub=viewer-1&name=Vera&groups=staff'   # who the next login is
+
+The issuer approves every login at once as the current user (the control endpoint changes it); `--issuer-file` receives the issuer URL for `auth.oidc.issuer`. **bcrypt PASSWORD** prints a bcrypt hash, the content of a break-glass file. Every devtools command that calls the API sends `Authorization: Bearer $NMON_TOKEN` when that variable is set (an nmon with authentication on needs an admin token).
+
+## Gates
+
+`make gate-m3` runs everything above end to end and writes live screenshots to `docs/screenshots/M3/`.
+It needs Playwright's chromium: `cd web && npx playwright install chromium`.
+
+`make gate-m6` starts nmon with a mock IdP and a break-glass file (authentication ON) and checks roles, tokens, break-glass lock-out, backup/restore, key rotation, settings reload, CSV import and the UI; its live screenshots go to `docs/screenshots/M6/`.

docs/milestones/M2-discovery-and-profiles.md

@@ -61,4 +61,25 @@ Adversarial: stack-member swap; RMA; GRE tunnel on Juniper; 32-bit wrap at 10G m
 
 ## Gate log
 
-_(filled in by the supervisor when the gate passes)_
+**Passed 2026-10-22**, run by the supervisor (Opus) on linux/amd64.
+
+| Check | Result |
+|---|---|
+| `make lint` (golangci-lint v2.14.0 + file sizes + ESLint) | **0 findings** (the maintainability hard rule took effect during M2: about 450 findings cleared, 3 justified `nolint`s for long OpenAPI doc tags) |
+| `go test -race ./...` and `-tags 'integration vmembed'` | all ok |
+| `./scripts/test-pg.sh` | ok (Postgres 16) |
+| Flaky-test proof | `go test -race -count=20 -run 'Concurrency\|Degraded\|Slow\|GlobalInflight' ./internal/poller/` ok |
+| `make gate-m1` | PASS (lint included) |
+| `make gate-m2` | PASS: 25 fixtures over IPv6 and IPv4 (13 profiles + behaviour fixtures), exact PDU budgets, 0 walk PDUs on the poll path; v1 (GETNEXT) and v3 authPriv via `nmon profile test` |
+| Real devices (`NMON_LIBRENMS_DIR`) | 12 LibreNMS recordings: every one matched the right profile, 0 type mismatches |
+
+**Notable decisions in M2**
+- MIBs are a sidecar, not embedded (DEVIATIONS 6). The LibreNMS importer is a separate offline tool (DEVIATIONS 9).
+- Identity: ambiguous matches open a review and freeze the candidates until an admin resolves it. Clean matches are audited only when something changed.
+- Polling: backoff and degraded start after 3 consecutive failed cycles. Lower interval groups are skipped while the hot group fails.
+- Text metrics are never polled; they are discovery label sources only.
+- A production timing race in the poller (a timer registered after the clock moved woke a full interval late) was found by the flaky-test investigation and fixed (`timerUntil`).
+
+**Bugs found by real-device testing and fixed:** apc-ups UPS-MIB label columns were not-accessible, and text metrics were polled as scalars. The fixture generator now refuses to emit not-accessible columns.
+
+**Carried forward:** identity-review resolve/merge endpoint (M3 API); `profile` `max_varbinds` key (deferred; poller option exists); a facets endpoint for the UI (M3).

docs/milestones/M3-graphs-and-ui.md

@@ -33,4 +33,36 @@ API and UI: openapi.json generated with the TS client and the incompatible-diff
 
 ## Gate log
 
-_(filled in by the supervisor when the gate passes)_
+**Passed 2026-10-22**, run by the supervisor (Opus).
+
+| Check | Result |
+|---|---|
+| `make lint` (Go + web) | 0 findings |
+| `go test -race ./...`, `-tags 'integration vmembed'`, `test-pg.sh` | all ok |
+| `make gate-m1`, `make gate-m2` (with LibreNMS recordings) | PASS |
+| `make gate-m3` | PASS: UI embedded in the static binary; SPA fallback and cache headers; 10 simulated devices (`nmon-devtools seed-demo`) with 31 days of back-filled history; port graph renders in and out at 1 h, 24 h, 30 d; 48-port page < 2 s; openapi-check; web client regenerated from the live spec type-checks |
+| UI ↔ API contract | `TestGeneratedSpecMatchesUIContract`: the generated `openapi.json` is compatible with the contract the UI was built against |
+| Screenshots | `docs/screenshots/M3/` (real data, 18 files) and `docs/screenshots/M3-mock/` (mock data); reviewed by the supervisor and sent to the owner |
+
+**Decisions in M3:** dev tooling is a separate `nmon-devtools` binary (owner's call), never shipped. `/docs` uses vendored Swagger UI. The two M2 provisional routes were reshaped before the route freeze, which starts at `m3`, the first tag with `openapi.json`. Graph stitching joins at now − hot retention; p95 is computed live until M5 materialises it.
+
+**Carried forward:** a demo device that is genuinely Down needs an unreachable address; that comes with M4's reachability work (`device-unresponsive.png` shows Warning/degraded for now, which is correct per policy because ICMP still answers). `nmon profile record` still links the simulator package; to be moved to a shared `internal/snmprec` format package.
+
+## API contract for M3 (written before the WPs so the UI and API are built in parallel)
+
+All JSON. Timestamps are RFC 3339 UTC. Lists use `?page=1&per_page=100` (max 1,000) and return `{items: [...], page, per_page, total}`. Errors are RFC 9457 `application/problem+json`. Existing M1/M2 shapes (`InterfaceView`, `DeviceView`, the `SNMPBody` input) are extended, never renamed.
+
+| Route | Response |
+|---|---|
+| `GET /api/v1/devices?site=&role=&kind=&q=&state=&page=&per_page=` | `items: DeviceSummary[]`, where `DeviceSummary{slug, name, kind, site, role, mgmt_ip, profile, reachability: up\|down\|warning\|unknown\|maintenance, poll_state: ok\|degraded\|backoff, last_seen, interfaces_total, interfaces_down, warnings[]}` |
+| `GET /api/v1/devices/{slug}` | `DeviceView` (existing) plus `parents[{slug,name}]`, `groups[{id,name,kind}]`, `sys_descr`, `sys_object_id`, `hw_serial`, `uptime_seconds`, `reachability`, `poll_state`, `warnings[]`, `interfaces: InterfaceView[]` with added `unreliable`, `retired_at`, and live fields `in_bps`, `out_bps`, `in_errors_ps`, `out_errors_ps` (5 min rate, null when unknown or unreliable) |
+| `PUT /api/v1/devices/{slug}` | body: `{name?, site?, role?, enabled?, parents?: [slug], snmp?: SNMPBody}`; returns `DeviceView` |
+| `GET /api/v1/interfaces?q=&page=` | `items: InterfaceHit[]`, where `InterfaceHit{id, device{slug,name}, ifname, ifalias, ifdescr, oper_status, admin_status, speed_bps, in_bps, out_bps}` (q matches ifname and ifalias, case-insensitive) |
+| `GET /api/v1/graph?iface=&from=&to=&step=` | `{iface{id,ifname,ifalias,speed_bps,device{slug,name}}, step_seconds, tier: raw\|1h, stitched_at?: ts, partial: [{from,to}], unreliable: bool, series: {in: [[unix_s, bps\|null]...], out: [...], in_max?: [...], out_max?: [...]}, p95?: {in_bps, out_bps, source: materialised\|live, coverage}}`. Both directions come in one call. `dir=` is optional and filters to one direction. An `unreliable` interface returns empty series and `unreliable_reason` (e.g. `counter_32bit_fast_link`). A stitched response also carries `tiers: [{from, to, tier, step_seconds}]`; the top-level `step_seconds` is the step of the newest tier. |
+| `GET /api/v1/poller/health` | `{devices: [{slug, name, state, varbinds, interval_s, base_interval_s, last_cycle_s, missed_deadlines, consecutive_failures, last_error, next_cycle}], totals: {pdus_per_min, timeouts_per_min, deadline_misses_per_min, write_queue_depth, write_queue_capacity, devices_degraded}}`. Sortable client-side. |
+| `GET /api/v1/identity-reviews?page=` and `POST /api/v1/identity-reviews/{id}/resolve` | review list with candidates and evidence; resolve body `{action: merge\|keep_both, keep?: iface_id}` |
+| `GET /api/v1/query`, `GET /api/v1/query_range` | (existing) |
+| `GET /api/v1/devices/facets` | `{sites: string[], roles: string[], kinds: string[]}`: distinct values over non-retired devices, for the filter dropdowns |
+| `GET /api/v1/meta` | `{version, build, retention{hot,fiveminute,long} (Go duration strings, e.g. "720h0m0s"), now}` for the UI footer and the range picker |
+
+**Graph tiers:** for a range ≤ hot retention, raw data from hot with step = max(60 s, range/1000). For longer ranges, hot raw data where it exists stitched with `nmon_if_rate_bps_avg_1h`/`max_1h` from long before that point; `stitched_at` is the join timestamp. Until M5's rollup exists, long has no data, so the response carries `partial` for the uncovered span. That is honest, not an error.

docs/milestones/M6-auth-and-ops.md

@@ -1,6 +1,6 @@
 # M6: Auth and ops
 
-**Gate:** a viewer gets 403 on PUT, and the 2,000-device loadtest cycle finishes under 40 s.
+**Gate:** a viewer gets 403 on PUT. ~~and the 2,000-device loadtest cycle finishes under 40 s~~ (skipped: approved deviation, see `docs/DEVIATIONS.md`; the owner prioritised getting to production).
 **Gate command:** `make gate-m6`. It runs the viewer-fuzz test (every state-changing route with a viewer session and a viewer token → 403) and then `nmon loadtest --devices 2000 --ports 100 --duration 30m`, which must report cycle p100 < 40 s, zero deadline misses, write-queue high-water < 50%, and stable RSS. It also prints the report metrics (peak RSS, steady and peak disk, ingest samples/s, p95 device-page latency, write-queue high-water, backlog recovery after a 5 min store outage). Finally it runs backup-restore during load.
 
 ## Scope
@@ -25,6 +25,11 @@
 - `nmon loadtest`: in-process simagents (2,000 × 100 ports on `::1` with distinct ports), nmon serve in-process, traps plus push plus rollup running, metrics collected from `nmon_` series and `/proc/self/status`.
 - Platforms: `make build-arm64` cross-compiles. Non-Linux builds compile without `vmembed` and require `--vm-*-url`.
 
+### Decisions recorded during M6 (see `docs/DEVIATIONS.md` rows 31-37)
+- **No login UI** (owner decision): page loads without a session redirect (302) to `/auth/login?return=...` (OIDC); API calls get 401; break-glass is `GET /auth/breakglass` with HTTP Basic; the SPA has a user menu (name, role, sign out) and role-aware controls but no login page. Dev-open mode is unchanged.
+- Hot reload covers receivers, the outbound allow-list, trusted proxies, backup schedule, disk thresholds and the purge period; the rest is `restart_required`.
+- Interfaces gained an operator-owned `description` (migration 8) that the CSV port import writes and discovery never overwrites.
+
 ## Work packages
 
 | WP | Files | Depends on |
@@ -32,7 +37,7 @@
 | WP1 OIDC, sessions, CSRF, tokens, break-glass, proxy, TLS | `internal/auth/**`, API middleware | M3 API |
 | WP2 Backup, restore, key rotate, disk guard, store restart | `internal/backup/**`, `internal/diskguard/**`, `cmd/nmon/{backup,restore,key,store,unlock}.go` | M1 vmsup, store |
 | WP3 Config reload, settings API and UI, CSV import, purge | `internal/config/**`, `internal/api/settings*.go`, `internal/importcsv/**`, `web/src/routes/settings*` | M4 deps |
-| WP4 Loadtest, viewer fuzz, gate | `internal/loadtest/**`, `cmd/nmon/loadtest.go`, `scripts/gate-m6.sh` | all |
+| WP4 Viewer fuzz, backup/restore check, gate (loadtest skipped, approved deviation) | `internal/loadtest/**`, `cmd/nmon/loadtest.go`, `scripts/gate-m6.sh` | all |
 
 ## Acceptance lines closed
 

docs/milestones/README.md

@@ -4,6 +4,7 @@ The build follows the six gated milestones in [`docs/spec/spec.md`](../spec/spec
 Each milestone has its own document here. It is written before any code for that milestone and
 its **Gate log** is filled in when the gate passes. A milestone is done only when:
 
+0. `make lint` is clean: maintainability is a hard rule ([`docs/MAINTAINABILITY.md`](../MAINTAINABILITY.md)), and every gate script runs lint first,
 1. every acceptance line it closes (from [`build-detail.md`](../spec/build-detail.md#acceptance-criteria)) has a test that passes,
 2. `make test` (`go test -race ./...`) and `make test-integration` pass,
 3. its gate command (`make gate-mN`) passes when run by the supervisor, and
@@ -12,11 +13,11 @@ its **Gate log** is filled in when the gate passes. A milestone is done only whe
 | # | Milestone | Gate | Doc | Status |
 |---|---|---|---|---|
 | M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | **passed**, tagged `m1` |
-| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | in progress (WP1 done) |
-| M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | planned |
-| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | planned |
+| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | **passed**, tagged `m2` |
+| M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | **passed**, tagged `m3` |
+| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | in progress (WP1 reach and deps, WP2 rules, WP3 notify done) |
 | M5 | Push ingest, rollup, checks | a replayed Proxmox sample shows under its node and a 2 y query hits the 1 h tier | [M5](M5-ingest-rollup-checks.md) | planned |
-| M6 | Auth and ops | a viewer gets 403 on PUT and the 2,000-device loadtest cycle finishes under 40 s | [M6](M6-auth-and-ops.md) | planned |
+| M6 | Auth and ops | a viewer gets 403 on PUT (2,000-device loadtest skipped, approved deviation) | [M6](M6-auth-and-ops.md) | planned |
 | M7 | gNMI streaming | after v1; not built in this run | — | out of scope |
 
 ## How the work is run
@@ -31,6 +32,8 @@ Departures from the spec are recorded in [`docs/DEVIATIONS.md`](../DEVIATIONS.md
 
 ## Rules every WP follows (from "Decisions that don't move")
 
+- **Maintainability is a hard rule.** Follow [`docs/MAINTAINABILITY.md`](../MAINTAINABILITY.md): zero golangci-lint and ESLint findings in your packages, the function, complexity and file-size limits, and the layering rules. Every WP brief cites it.
+
 - One static binary, `CGO_ENABLED=0`. No raw SQL outside `internal/store`.
 - The 60 s poll path talks only to devices (SNMP, ICMP) and the local hot store.
 - Metric labels on SNMP series: `device=<slug>`, plus `iface=<ULID>` on interface series. Nothing else.
@@ -39,3 +42,13 @@ Departures from the spec are recorded in [`docs/DEVIATIONS.md`](../DEVIATIONS.md
 - IPv6 first: every network path works on v6 and v4; tests default to `::1`.
 - Time comes from `internal/clock`. No `time.Now()`/`time.Sleep` in logic that a test must control.
 - Tests never need real hardware. Fixtures are `.snmprec` files served by `internal/simagent`.
+
+## Deferred (revisit after v1 milestones)
+
+- **Deployment playbook**: install layout (binary, `/usr/share/nmon/mibs` sidecar, systemd unit granting port 162, key file location), TLS/ACME or proxy setup, IdP configuration, backups, and upgrades. Raised by the user on 2026-10-22; deliberately postponed.
+
+- **Web push notifications (v2)**: a `webpush` receiver kind in `internal/notify` (RFC 8291 payload encryption, RFC 8292 VAPID, standard-library crypto), per-user subscriptions (needs M6 users), a service worker in the UI, and push endpoints in the outbound allow-list. Out of v1 per the spec's non-goals ("alerts go to webhooks and email only"). Owner deferred it to v2 on 2026-10-22. Until then, a webhook receiver pointed at ntfy or Gotify gives phone push.
+
+## Review artifacts
+
+- From M3 on, each UI milestone's gate captures Playwright screenshots of every page into `docs/screenshots/<milestone>/` for the user to review.

docs/operations.md

@@ -0,0 +1,68 @@
+# Operating nmon
+
+How to run, secure, back up and maintain a production nmon. Design background: `docs/spec/spec.md` ("Auth", "Operations").
+
+## Starting
+
+    nmon serve --data-dir /var/lib/nmon --secret-key-file /etc/nmon/secret.key \
+               --tls-cert /etc/nmon/tls.crt --tls-key /etc/nmon/tls.key
+
+- **HTTPS is the default.** Pass a certificate pair (`--tls-cert`, `--tls-key`; `SIGHUP` reloads them without dropping connections) or Let's Encrypt (`--acme-host nmon.example.net` repeatable, `--acme-email you@example.net`; nmon answers the http-01 challenge on `--acme-listen`, default `:80`, certificates are cached in `<data-dir>/acme`). Without either, `serve` refuses to start and says so. `--insecure-http` is for development: plain HTTP, cookies without `Secure`.
+- **A sign-in method is required** over TLS (`auth.oidc` and/or `auth.break_glass_file` in `config.yaml`). With `--insecure-http` and neither configured nmon runs in *dev-open mode* (everyone is admin `dev`; a banner says so); never expose that port.
+- Behind a reverse proxy, list it in `auth.trusted_proxies`; `X-Forwarded-*` is ignored from anyone else.
+
+## Sign-in
+
+`config.yaml`:
+
+    auth:
+      oidc:
+        issuer: https://idp.example.net/realms/net
+        client_id: nmon
+        client_secret_name: oidc-client      # the secret itself is never in this file
+        redirect_url: https://nmon.example.net/auth/callback
+        groups_claim: groups
+        admin_group: nmon-admins             # members are admins; every other user is a viewer
+      break_glass_file: /etc/nmon/breakglass # bcrypt hash, "hash" or "user:hash", created by you
+
+Store the client secret before the first start: `printf '%s\n' "$SECRET" | nmon secret set oidc-client --data-dir D --secret-key-file K` (it can also be set later with `PUT /api/v1/secrets/oidc-client`; OIDC stays off until the secret exists). The break-glass file holds one bcrypt hash (`$2a$`/`$2b$`), made with any bcrypt tool (the source tree's `nmon-devtools bcrypt PASSWORD` works); keep it readable by the nmon user only.
+
+- There is no login page. Opening nmon without a session redirects to the IdP and back; API clients use a token. If the IdP is down, open `/auth/breakglass`: the browser asks for the local admin user and password (HTTP Basic), and a successful sign-in lands on the page you asked for.
+- Break-glass: five wrong passwords lock the source address for 15 minutes; every use and every lock is audited and raises an alert. `nmon unlock [IP|all] --data-dir D` clears locks, also while nmon runs.
+- API tokens: an admin mints viewer, admin or push tokens (`POST /api/v1/tokens`, or Settings, API tokens). Use `Authorization: Bearer nma_...`. A push token works for ingest only. Revoking takes effect on the next request.
+- `GET /api/v1/audit` (admin) lists who did what.
+
+## Backup and restore
+
+A running nmon owns the stores, so copying the data folder is **not** a backup.
+
+    nmon backup /backups/nmon.tar --api https://nmon.example.net --token nma_...   # key NOT included
+    nmon backup /backups/nmon.tar --include-key --api ... --token ...             # key included
+
+Every run prints whether the key file is in the archive. **Lose the key file and a restore has an inventory with no SNMP credentials.** The nightly schedule is `backup: {at: "02:30", dir: /var/backups/nmon, keep: 7}` (time in the configured timezone, default dir `<data-dir>/backups`); a failed run raises the `backup_failed` alert. A backup is refused when the disk is below its 5% reserve.
+
+Restore with nmon stopped, into a clean directory:
+
+    nmon restore /backups/nmon.tar --data-dir /var/lib/nmon-restored --secret-key-file /etc/nmon/secret.key
+
+It verifies inventory row counts and a sample of series before it says done. Then start nmon on that directory. Restore needs the embedded stores (the normal release build).
+
+## Keys
+
+`nmon key rotate --data-dir D --secret-key-file K` (nmon stopped) adds a new key version, re-encrypts every SNMP credential and secret under it, and keeps the old key in the file so older backups still restore. Remove old keys only after a release.
+
+## Configuration
+
+Edit in Settings (admin) or `config.yaml` + `SIGHUP`. An invalid document is rejected and the running configuration stays. Alert receivers, the outbound allow-list, trusted proxies, the backup schedule, disk thresholds and the purge period apply live; everything else is listed as `restart_required` until the next restart. `disk: {alert_used: 0.8, admit_used: 0.9, reserve_free: 0.05}`: alert at 80% used, no new series at 90%, ingest throttled / rollup paused / backups refused below 5% free.
+
+## Metric stores
+
+The supervisor restarts a crashed store; after three restarts in ten minutes it raises `store_failed` and stops. `nmon store restart [hot|long] --api URL --token T` (or `SIGHUP`) resumes it.
+
+## Retired devices
+
+`POST /api/v1/devices/{slug}/purge` (admin) deletes a retired device's series from both stores once it has been retired for `retired_purge_after` (default 1y).
+
+## CSV import
+
+Devices (`name,address,site,role,credential,parents,...`) and port descriptions (`device,ifname,alias`) from Settings or `POST /api/v1/devices/import`. Port descriptions are stored in the operator-owned `description` field, which discovery never overwrites. A dependency cycle is rejected with its path; nothing is written unless the whole import is valid.