M3: Go API per UI contract, embedded UI, openapi compat check, nmon-devtools, live gate + screenshots; M4 WP1 reachability and dependency suppression

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP

.gitignore

@@ -1,5 +1,6 @@
 /nmon
 /nmon-arm64
+/nmon-devtools
 /internal/vmsup/bin/*.tar.gz
 /internal/vmsup/bin/victoria-metrics-*
 web/node_modules/
@@ -8,3 +9,5 @@ web/dist/
 *.test
 coverage.out
 /nmon-librenms-import
+/internal/webui/dist/*
+!/internal/webui/dist/.gitkeep

.golangci.yml

@@ -107,6 +107,21 @@ linters:
               desc: collection paths share code via a lower package, not each other
             - pkg: github.com/v10networks/nmon/internal/traps
               desc: collection paths share code via a lower package, not each other
+        devtools-not-in-nmon:
+          # profile_record.go (`nmon profile record`, M2) already uses the recording types and writer; nothing else may.
+          files: ["**/cmd/nmon/**", "!**/cmd/nmon/profile_record.go", "!$test"]
+          deny:
+            - pkg: github.com/v10networks/nmon/internal/simagent
+              desc: the simulator belongs to cmd/nmon-devtools and tests, never the shipped binary
+            - pkg: github.com/v10networks/nmon/internal/fixturegen
+              desc: the fixture generator belongs to cmd/nmon-devtools and tests, never the shipped binary
+        devtools-no-core:
+          files: ["**/cmd/nmon-devtools/**"]
+          deny:
+            - pkg: github.com/v10networks/nmon/internal/app
+              desc: nmon-devtools talks to a running nmon over HTTP
+            - pkg: github.com/v10networks/nmon/internal/api
+              desc: nmon-devtools talks to a running nmon over HTTP
         importer-not-in-nmon:
           files: ["**/cmd/nmon/**"]
           deny:

Makefile

@@ -1,24 +1,33 @@
 GO        ?= go
 TAGS      ?= vmembed
-LDFLAGS   := -s -w
+VERSION   ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
+BUILD     ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)-$(shell date -u +%Y%m%d)
+LDFLAGS   := -s -w -X github.com/v10networks/nmon/internal/buildinfo.Version=$(VERSION) -X github.com/v10networks/nmon/internal/buildinfo.Build=$(BUILD)
 BIN       := nmon
 PG_DSN    ?= postgres://nmon:nmon@localhost:55432/nmon?sslmode=disable
 
-.PHONY: all lint lint-go lint-web build importer build-arm64 build-noembed fetch-vm web test test-integration test-pg test-v6only vet gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6 clean
+.PHONY: devtools all lint lint-go lint-web build importer build-arm64 build-noembed fetch-vm web test test-integration test-pg test-v6only vet gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6 clean
 
 all: build
 
 fetch-vm:
 	./scripts/fetch-vm.sh
 
-# The UI is built first when web/ exists so embed.FS picks it up.
+# The UI is built first, then copied into internal/webui/dist for go:embed (git-ignored).
 web:
-	@if [ -f web/package.json ]; then cd web && npm ci --no-audit --no-fund && npm run build; fi
+	@if [ -f web/package.json ]; then \
+		cd web && { [ -d node_modules ] || npm ci --no-audit --no-fund; } && npm run build && \
+		cd .. && rm -rf internal/webui/dist/* && cp -r web/dist/. internal/webui/dist/ && touch internal/webui/dist/.gitkeep; \
+	fi
 
-build: fetch-vm
+build: fetch-vm web
 	CGO_ENABLED=0 $(GO) build -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(BIN) ./cmd/nmon
 
-build-arm64: fetch-vm
+# Developer tools (backfill, seed-demo) are a separate binary that is never shipped.
+devtools:
+	CGO_ENABLED=0 $(GO) build -o nmon-devtools ./cmd/nmon-devtools
+
+build-arm64: fetch-vm web
 	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(GO) build -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(BIN)-arm64 ./cmd/nmon
 
 importer:
@@ -59,4 +68,4 @@ gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6: gate-m%:
 	./scripts/gate-m$*.sh
 
 clean:
-	rm -f $(BIN) $(BIN)-arm64 nmon-librenms-import
+	rm -f $(BIN) $(BIN)-arm64 nmon-devtools nmon-librenms-import

cmd/nmon-devtools/backfill.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/v10networks/nmon/internal/metrics"
+)
+
+// backfillConfig is what one `devtools backfill` run writes, for every interface in ifaces.
+type backfillConfig struct {
+	hotURL  string
+	device  string
+	ifaces  []string
+	days    int
+	step    time.Duration
+	pattern string
+	meanBps float64 // bits/s; 0 derives the mean from the live rate, else a stable per-interface value
+	seed    uint64
+	hc      *http.Client
+	logf    func(format string, a ...any)
+}
+
+func backfillCmd(ctx context.Context, args []string) error {
+	cfg := backfillConfig{hc: &http.Client{Timeout: 2 * time.Minute}, logf: func(f string, a ...any) { fmt.Printf(f+"\n", a...) }}
+	fs := devFlagSet("backfill")
+	var ifaces stringList
+	var mbps float64
+	fs.StringVar(&cfg.hotURL, "hot-url", "", "base URL of the hot VictoriaMetrics (required; see \"metric stores up\" in the serve log)")
+	fs.StringVar(&cfg.device, "device", "", "device slug (the series' device label)")
+	fs.Var(&ifaces, "iface", "interface ULID; repeat the flag or separate with commas")
+	fs.IntVar(&cfg.days, "days", 31, "days of history to write, ending where the live data begins")
+	fs.DurationVar(&cfg.step, "step", 60*time.Second, "sample spacing")
+	fs.StringVar(&cfg.pattern, "pattern", patternDiurnal, "traffic shape: diurnal or flat")
+	fs.Float64Var(&mbps, "mean-mbps", 0, "mean rate in Mbit/s; 0 follows the live rate when there is one")
+	fs.Uint64Var(&cfg.seed, "seed", 1, "noise seed")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	cfg.ifaces, cfg.meanBps = ifaces, mbps*1e6
+	if err := cfg.validate(); err != nil {
+		return err
+	}
+	for _, id := range cfg.ifaces {
+		if err := cfg.backfillIface(ctx, id); err != nil {
+			return fmt.Errorf("iface %s: %w", id, err)
+		}
+	}
+	return nil
+}
+
+func (c *backfillConfig) validate() error {
+	switch {
+	case c.hotURL == "":
+		return errors.New("--hot-url is required")
+	case c.device == "" || len(c.ifaces) == 0:
+		return errors.New("--device and at least one --iface are required")
+	case c.days < 1 || c.step < time.Second:
+		return errors.New("--days must be at least 1 and --step at least 1s")
+	case c.pattern != patternDiurnal && c.pattern != patternFlat:
+		return fmt.Errorf("unknown --pattern %q (diurnal or flat)", c.pattern)
+	}
+	c.hotURL = strings.TrimRight(c.hotURL, "/")
+	return nil
+}
+
+// liveSeries is what the hot store already holds for one series: the first
+// sample (where history must end and which value it must join) and the live rate.
+type liveSeries struct {
+	ts   time.Time
+	val  float64
+	rate float64 // units per second between the first two samples; 0 when unknown
+	ok   bool
+}
+
+// backfillIface writes in/out octet counters and the oper status for one interface.
+func (c *backfillConfig) backfillIface(ctx context.Context, iface string) error {
+	n := c.days * int(24*time.Hour/c.step)
+	var samples []metrics.Sample
+	var end time.Time
+	for d, dir := range []struct {
+		metric string
+		salt   uint64
+	}{{"ifHCInOctets", 0x1111}, {"ifHCOutOctets", 0x2222}} {
+		live, err := c.probe(ctx, dir.metric, iface)
+		if err != nil {
+			return err
+		}
+		if d == 0 {
+			end = c.endTime(live)
+		}
+		samples = append(samples, c.counter(dir.metric, iface, end, n, live, c.seed^dir.salt)...)
+	}
+	status := 1.0
+	if live, err := c.probe(ctx, "ifOperStatus", iface); err == nil && live.ok {
+		status = live.val
+	}
+	for i := 0; i < n; i++ {
+		ts := end.Add(-time.Duration(n-1-i) * c.step)
+		samples = append(samples, metrics.New("ifOperStatus", status, ts, metrics.LabelDevice, c.device, metrics.LabelIface, iface))
+	}
+	if err := c.send(ctx, samples); err != nil {
+		return err
+	}
+	c.logf("backfilled %s/%s: %d days at %s (%d samples)", c.device, iface, c.days, c.step, len(samples))
+	return nil
+}
+
+// endTime is one step before the first live sample, or one step ago when there is none.
+func (c *backfillConfig) endTime(live liveSeries) time.Time {
+	if live.ok {
+		return live.ts.Add(-c.step).Truncate(time.Millisecond)
+	}
+	return time.Now().Add(-c.step).Truncate(time.Millisecond)
+}
+
+// counter builds one direction's history ending at end. When the live counter
+// is known and large enough, the history is shifted so its last value is where
+// the live counter was one step earlier and the join shows no reset.
+func (c *backfillConfig) counter(metric, iface string, end time.Time, n int, live liveSeries, salt uint64) []metrics.Sample {
+	mean := c.meanBps / 8
+	if mean == 0 && live.rate > 0 {
+		mean = live.rate
+	}
+	if mean == 0 {
+		mean = defaultMeanBytesPerSec(iface, salt)
+	}
+	start := end.Add(-time.Duration(n-1) * c.step)
+	vals := counterSeries(c.pattern, start, n, c.step, mean, salt)
+	var offset float64
+	if live.ok {
+		offset = live.val - mean*c.step.Seconds() - vals[n-1]
+		if offset < 0 {
+			c.logf("warning: %s %s: live counter %.3g is below the %.3g bytes of history; the join will look like a counter reset",
+				c.device, metric, live.val, vals[n-1])
+			offset = 0
+		}
+	}
+	out := make([]metrics.Sample, n)
+	for i, v := range vals {
+		ts := start.Add(time.Duration(i) * c.step)
+		out[i] = metrics.New(metric, math.Floor(v+offset), ts, metrics.LabelDevice, c.device, metrics.LabelIface, iface)
+	}
+	return out
+}
+
+// probe reads the earliest samples of metric{iface=...} from the hot store.
+func (c *backfillConfig) probe(ctx context.Context, metric, iface string) (liveSeries, error) {
+	q := url.Values{"match[]": {fmt.Sprintf(`%s{iface=%q}`, metric, iface)}}
+	var raw bytes.Buffer
+	if err := c.get(ctx, c.hotURL+"/api/v1/export?"+q.Encode(), &raw); err != nil {
+		return liveSeries{}, err
+	}
+	return parseExport(raw.Bytes())
+}

cmd/nmon-devtools/fleet.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/gosnmp/gosnmp"
+
+	"github.com/v10networks/nmon/internal/fixturegen"
+	"github.com/v10networks/nmon/internal/profile"
+	"github.com/v10networks/nmon/internal/simagent"
+)
+
+// demoDevice is one simulated device of `devtools seed-demo`.
+type demoDevice struct {
+	name, site, role string
+	kind             string // counters, big, router, ups, v1, down: what the gate and screenshots look for
+	rec              *simagent.Recording
+	community        string
+	version          string // v2c unless set
+	hc               bool   // 64-bit counters, so history can be back-filled
+}
+
+// counterBase is added to every simulated 64-bit octet counter so that the
+// back-filled history (about 3e13 bytes for 31 days at 100 Mbit/s) fits below
+// the live counter and the join shows no reset.
+const counterBase = 100_000_000_000_000
+
+const (
+	oidHCIn  = "1.3.6.1.2.1.31.1.1.1.6."
+	oidHCOut = "1.3.6.1.2.1.31.1.1.1.10."
+	oidUp    = "1.3.6.1.2.1.1.3.0"
+)
+
+// animate returns a copy of rec whose 64-bit octet counters advance at a steady
+// per-port rate from counterBase and whose uptime ticks, so successive polls see
+// traffic. Counters that already advance keep their rate and only gain the base.
+func animate(rec *simagent.Recording) *simagent.Recording {
+	recs := append([]simagent.Record(nil), rec.Records()...)
+	for i := range recs {
+		r := &recs[i]
+		switch {
+		case r.Type == gosnmp.Counter64 && (strings.HasPrefix(r.OID, oidHCIn) || strings.HasPrefix(r.OID, oidHCOut)):
+			recs[i] = animateCounter(*r)
+		case r.OID == oidUp && r.Dyn == nil:
+			if v, ok := r.Value.(uint32); ok {
+				r.Dyn, r.Value = &simagent.Dynamic{Rate: 100, Initial: uint64(v)}, nil
+			}
+		}
+	}
+	return simagent.NewRecording(recs)
+}
+
+func animateCounter(r simagent.Record) simagent.Record {
+	if r.Dyn != nil {
+		d := *r.Dyn
+		d.Initial += counterBase
+		r.Dyn = &d
+		return r
+	}
+	k, _ := strconv.Atoi(r.OID[strings.LastIndexByte(r.OID, '.')+1:])
+	dir := 1
+	if strings.HasPrefix(r.OID, oidHCOut) {
+		dir = 2
+	}
+	mbps := 1 + (k*(31+7*dir))%90
+	r.Dyn = &simagent.Dynamic{Rate: float64(mbps) * 125_000, Initial: counterBase + uint64(k)*1_000_000_000}
+	r.Value = nil
+	return r
+}
+
+// profileDevice builds a device that a shipped profile matches, with traffic.
+func profileDevice(set *profile.Set, prof, name, site, role, kind string, ports int) (demoDevice, error) {
+	spec, ok := fixturegen.SpecFor(prof)
+	if !ok {
+		return demoDevice{}, fmt.Errorf("no fixture spec for profile %q", prof)
+	}
+	res, ok := set.Lookup(prof)
+	if !ok {
+		return demoDevice{}, fmt.Errorf("no profile %q", prof)
+	}
+	spec.SysName = name
+	if ports > 0 {
+		spec.Ports = ports
+	}
+	rec, err := fixturegen.FromProfile(res, spec)
+	if err != nil {
+		return demoDevice{}, fmt.Errorf("%s: %w", name, err)
+	}
+	return demoDevice{name: name, site: site, role: role, kind: kind, rec: animate(rec), community: "public", hc: true}, nil
+}
+
+// buildFleet is the small network used for live screenshots: a 48-port switch,
+// routers, a firewall, a UPS, one 32-bit device, the counters fixture and one
+// device whose agent is stopped after seeding, so it reads as down.
+func buildFleet(set *profile.Set, countersRec string) ([]demoDevice, error) {
+	counters, err := simagent.LoadRecording(countersRec)
+	if err != nil {
+		return nil, err
+	}
+	fleet := []demoDevice{{
+		name: "yyz1-edge1", site: "yyz1", role: "edge", kind: "counters", rec: animate(counters), community: "public", hc: true,
+	}}
+	for _, p := range []struct {
+		prof, name, site, role, kind string
+		ports                        int
+	}{
+		{"cisco-ios", "yyz1-acc01", "yyz1", "access", "big", 48},
+		{"cisco-ios", "yyz1-acc02", "yyz1", "access", "router", 24},
+		{"cisco-ios", "ott1-acc02", "ott1", "access", "down", 24}, // its agent is stopped once seeding is done
+		{"juniper-junos", "yyz1-core1", "yyz1", "core", "router", 0},
+		{"arista-eos", "yul1-core1", "yul1", "core", "router", 0},
+		{"mikrotik-routeros", "ott1-gw1", "ott1", "gateway", "router", 0},
+		{"fortigate", "yyz1-fw1", "yyz1", "firewall", "router", 0},
+		{"apc-ups", "yyz1-ups1", "yyz1", "power", "ups", 0},
+	} {
+		d, err := profileDevice(set, p.prof, p.name, p.site, p.role, p.kind, p.ports)
+		if err != nil {
+			return nil, err
+		}
+		fleet = append(fleet, d)
+	}
+	// 32-bit counters on 1 Gbit/s ports: the API flags them unreliable and the UI never graphs them.
+	v1 := fixturegen.Generic(fixturegen.Options{Ports: 12, SysName: "yul1-acc03", Counters: true, NamePattern: "Fa0/%d"})
+	fleet = append(fleet, demoDevice{
+		name: "yul1-acc03", site: "yul1", role: "access", kind: "v1", rec: v1, community: "nmon-v1", version: "v1",
+	})
+	return fleet, nil
+}

cmd/nmon-devtools/main.go

@@ -0,0 +1,119 @@
+// Command nmon-devtools holds developer helpers for the gates and for screenshots: backfill (synthetic
+// history into the hot store) and seed-demo (a simulated network added to a running nmon). It talks to nmon
+// only through its HTTP API and to the hot store through --hot-url, and is never shipped (docs/dev.md).
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+)
+
+var errUsage = errors.New("usage: nmon-devtools backfill|seed-demo [flags] (see docs/dev.md)")
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	if err := run(ctx, os.Args[1:]); err != nil {
+		fmt.Fprintln(os.Stderr, "nmon-devtools:", err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, args []string) error {
+	if len(args) == 0 {
+		return errUsage
+	}
+	switch args[0] {
+	case "backfill":
+		return backfillCmd(ctx, args[1:])
+	case "seed-demo":
+		return seedDemoCmd(ctx, args[1:])
+	default:
+		return fmt.Errorf("unknown command %q: %w", args[0], errUsage)
+	}
+}
+
+// flagErr turns flag.ErrHelp into a clean exit.
+func flagErr(err error) error {
+	if errors.Is(err, flag.ErrHelp) {
+		os.Exit(0)
+	}
+	return err
+}
+
+// stringList is a repeatable string flag.
+type stringList []string
+
+func (s *stringList) String() string { return strings.Join(*s, ",") }
+
+func (s *stringList) Set(v string) error {
+	for _, p := range strings.Split(v, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			*s = append(*s, p)
+		}
+	}
+	return nil
+}
+
+// devFlagSet is a FlagSet that prints its own usage to stderr.
+func devFlagSet(name string) *flag.FlagSet {
+	fs := flag.NewFlagSet("nmon-devtools "+name, flag.ContinueOnError)
+	fs.SetOutput(os.Stderr)
+	return fs
+}
+
+// httpJSON sends an optional JSON body and decodes a 2xx JSON reply into out (nil to discard it).
+func httpJSON(ctx context.Context, hc *http.Client, method, url string, in, out any) error {
+	var body io.Reader
+	if in != nil {
+		b, err := json.Marshal(in)
+		if err != nil {
+			return err
+		}
+		body = strings.NewReader(string(b))
+	}
+	req, err := http.NewRequestWithContext(ctx, method, url, body)
+	if err != nil {
+		return err
+	}
+	if in != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	resp, err := hc.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }() // body read to the end below
+	data, err := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("%s %s: %s: %s", method, url, resp.Status, strings.TrimSpace(string(data[:min(len(data), 300)])))
+	}
+	if out == nil {
+		return nil
+	}
+	return json.Unmarshal(data, out)
+}
+
+func sleepCtx(ctx context.Context, d time.Duration) bool {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+		return false
+	case <-t.C:
+		return true
+	}
+}

cmd/nmon-devtools/pattern.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"math"
+	"time"
+)
+
+// Synthetic traffic shapes for `nmon devtools backfill`. Everything here is a
+// pure function of its inputs, so a back-fill is reproducible and testable.
+
+const (
+	patternDiurnal = "diurnal"
+	patternFlat    = "flat"
+)
+
+// splitmix64 is a tiny deterministic hash used for noise; math/rand would make
+// two runs differ and a global source would be shared state.
+func splitmix64(x uint64) uint64 {
+	x += 0x9e3779b97f4a7c15
+	x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9
+	x = (x ^ (x >> 27)) * 0x94d049bb133111eb
+	return x ^ (x >> 31)
+}
+
+// unit maps a hash to [0, 1).
+func unit(h uint64) float64 { return float64(h>>11) / (1 << 53) }
+
+// rateFactor is the multiple of the mean rate at t: 1 for "flat"; for
+// "diurnal" an evening peak (about 20:00 UTC), an early-morning trough, quieter
+// weekends, per-sample noise and the odd short burst.
+func rateFactor(pattern string, t time.Time, salt uint64) float64 {
+	if pattern == patternFlat {
+		return 1
+	}
+	h := float64(t.Hour()) + float64(t.Minute())/60
+	f := 1 + 0.6*math.Cos(2*math.Pi*(h-20)/24)
+	if wd := t.Weekday(); wd == time.Saturday || wd == time.Sunday {
+		f *= 0.8
+	}
+	slot := uint64(t.Unix() / 60)
+	f *= 0.88 + 0.24*unit(splitmix64(salt^slot))
+	if unit(splitmix64(salt^(slot/4)^0xb5ad4eceda1ce2a9)) < 0.01 { // ~1% of 4-minute slots burst
+		f *= 2.5
+	}
+	return math.Max(f, 0.05)
+}
+
+// counterSeries returns n cumulative octet-counter values, one per step from
+// start, beginning at 0 and never decreasing. mean is bytes per second.
+func counterSeries(pattern string, start time.Time, n int, step time.Duration, mean float64, salt uint64) []float64 {
+	out := make([]float64, n)
+	var total float64
+	for i := 1; i < n; i++ {
+		t := start.Add(time.Duration(i) * step)
+		total += mean * rateFactor(pattern, t, salt) * step.Seconds()
+		out[i] = math.Floor(total)
+	}
+	return out
+}
+
+// defaultMeanBytesPerSec is a stable per-interface mean (2 to 60 Mbit/s) for when
+// the live rate is unknown.
+func defaultMeanBytesPerSec(iface string, salt uint64) float64 {
+	h := salt
+	for _, c := range []byte(iface) {
+		h = splitmix64(h ^ uint64(c))
+	}
+	return (2 + 58*unit(h)*unit(h>>7)) * 125_000
+}

cmd/nmon-devtools/pattern_test.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCounterSeriesIsMonotonicAndDeterministic(t *testing.T) {
+	start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
+	a := counterSeries(patternDiurnal, start, 3*24*60, time.Minute, 1_000_000, 7)
+	b := counterSeries(patternDiurnal, start, 3*24*60, time.Minute, 1_000_000, 7)
+	for i := range a {
+		if a[i] != b[i] {
+			t.Fatalf("not deterministic at %d", i)
+		}
+		if i > 0 && a[i] < a[i-1] {
+			t.Fatalf("decreased at %d", i)
+		}
+	}
+	if a[0] != 0 {
+		t.Fatalf("starts at %v, want 0", a[0])
+	}
+	meanRate := a[len(a)-1] / (float64(len(a)-1) * 60)
+	if meanRate < 0.8e6 || meanRate > 1.4e6 {
+		t.Fatalf("mean rate %.0f B/s drifted from 1e6", meanRate)
+	}
+}
+
+func TestDiurnalEveningBusierThanNight(t *testing.T) {
+	// Monday, so the weekend factor does not interfere. Averaged over many days the noise washes out.
+	var night, evening float64
+	for d := 0; d < 28; d++ {
+		day := time.Date(2026, 9, 7, 0, 0, 0, 0, time.UTC).AddDate(0, 0, d)
+		if wd := day.Weekday(); wd == time.Saturday || wd == time.Sunday {
+			continue
+		}
+		night += rateFactor(patternDiurnal, day.Add(5*time.Hour), 1)
+		evening += rateFactor(patternDiurnal, day.Add(20*time.Hour), 1)
+	}
+	if evening < 2*night {
+		t.Fatalf("evening %.2f is not clearly busier than night %.2f", evening, night)
+	}
+}
+
+func TestFlatPatternIsConstant(t *testing.T) {
+	if f := rateFactor(patternFlat, time.Now(), 3); f != 1 {
+		t.Fatalf("flat factor %v", f)
+	}
+}
+
+func TestDefaultMeanIsStableAndBounded(t *testing.T) {
+	m := defaultMeanBytesPerSec("01abc", 1)
+	if m != defaultMeanBytesPerSec("01abc", 1) || m < 2*125_000 || m > 60*125_000 {
+		t.Fatalf("mean %v", m)
+	}
+}

cmd/nmon-devtools/seed.go

@@ -0,0 +1,242 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/v10networks/nmon/internal/profile"
+	"github.com/v10networks/nmon/internal/simagent"
+)
+
+// seedConfig is the command line of `devtools seed-demo`.
+type seedConfig struct {
+	api, hotURL string
+	days        int
+	step        time.Duration
+	countersRec string
+	envOut      string
+	profilesDir string
+	workers     int
+	hc          *http.Client
+}
+
+// seeded is one device after it was added to nmon.
+type seeded struct {
+	demoDevice
+	slug   string
+	agent  *simagent.Agent
+	ip     string
+	ifaces []seedIface
+}
+
+type seedIface struct {
+	ID         string `json:"id"`
+	Name       string `json:"ifname"`
+	HC         bool   `json:"has_hc_counters"`
+	Excluded   bool   `json:"excluded"`
+	Unreliable bool   `json:"unreliable"`
+}
+
+func (i seedIface) graphable() bool { return i.HC && !i.Excluded && !i.Unreliable }
+
+func seedDemoCmd(ctx context.Context, args []string) error {
+	cfg := seedConfig{hc: &http.Client{Timeout: 3 * time.Minute}}
+	fs := devFlagSet("seed-demo")
+	fs.StringVar(&cfg.api, "api", "http://127.0.0.1:8443", "base URL of the running nmon (started with --insecure-http)")
+	fs.StringVar(&cfg.hotURL, "hot-url", "", "base URL of the hot VictoriaMetrics, for the back-fill (required)")
+	fs.IntVar(&cfg.days, "days", 31, "days of history to back-fill")
+	fs.DurationVar(&cfg.step, "step", 60*time.Second, "back-fill sample spacing")
+	fs.StringVar(&cfg.countersRec, "counters-rec", "testdata/snmprec/counters.snmprec", "the counters fixture recording")
+	fs.StringVar(&cfg.envOut, "env-out", "", "write NMON_DEVICE=<counters slug> and NMON_DEVICE_48=<48-port slug> to this file when ready")
+	fs.StringVar(&cfg.profilesDir, "profiles-dir", "", "operator profiles directory (shipped profiles are always loaded)")
+	fs.IntVar(&cfg.workers, "workers", 6, "parallel back-fill workers")
+	if err := fs.Parse(args); err != nil {
+		return flagErr(err)
+	}
+	if cfg.hotURL == "" {
+		return errors.New("--hot-url is required")
+	}
+	cfg.api, cfg.hotURL = strings.TrimRight(cfg.api, "/"), strings.TrimRight(cfg.hotURL, "/")
+	return cfg.run(ctx)
+}
+
+func (c *seedConfig) run(ctx context.Context) error {
+	set, err := profile.Load(c.profilesDir)
+	if err != nil {
+		return err
+	}
+	fleet, err := buildFleet(set, c.countersRec)
+	if err != nil {
+		return err
+	}
+	var pool simagent.Pool
+	defer func() { _ = pool.Close() }() // process is exiting
+	added, err := c.addAll(ctx, &pool, fleet)
+	if err != nil {
+		return err
+	}
+	if err := c.backfillAll(ctx, added); err != nil {
+		return err
+	}
+	if err := c.writeEnv(added); err != nil {
+		return err
+	}
+	fmt.Println("seed-demo ready: agents keep answering until interrupted")
+	<-ctx.Done()
+	return nil
+}
+
+// addAll starts one agent per device on [::1] with its own port and adds the
+// device through the API with that port.
+func (c *seedConfig) addAll(ctx context.Context, pool *simagent.Pool, fleet []demoDevice) ([]seeded, error) {
+	var out []seeded
+	for i, d := range fleet {
+		cfg := simagent.Config{Addr: "[::1]:0", Recording: d.rec, Community: d.community}
+		if err := pool.Start(ctx, []simagent.Config{cfg}); err != nil {
+			return nil, fmt.Errorf("%s: %w", d.name, err)
+		}
+		agent := pool.Agents()[i]
+		s, err := c.addDevice(ctx, d, "::1", int(agent.Addr().Port()))
+		s.agent = agent
+		if err != nil {
+			return nil, fmt.Errorf("add %s: %w", d.name, err)
+		}
+		fmt.Printf("added %-12s %-10s %s (%d interfaces)\n", d.name, d.kind, s.slug, len(s.ifaces))
+		out = append(out, s)
+	}
+	return out, nil
+}
+
+func (c *seedConfig) addDevice(ctx context.Context, d demoDevice, ip string, port int) (seeded, error) {
+	version := d.version
+	if version == "" {
+		version = "v2c"
+	}
+	body := map[string]any{
+		"name": d.name, "addresses": []string{ip}, "site": d.site, "role": d.role,
+		"snmp": map[string]any{"version": version, "community": d.community, "port": port},
+	}
+	var created struct {
+		Slug string `json:"slug"`
+	}
+	if err := httpJSON(ctx, c.hc, http.MethodPost, c.api+"/api/v1/devices", body, &created); err != nil {
+		return seeded{}, err
+	}
+	s := seeded{demoDevice: d, slug: created.Slug, ip: ip}
+	var got struct {
+		Interfaces []seedIface `json:"interfaces"`
+	}
+	if err := httpJSON(ctx, c.hc, http.MethodGet, c.api+"/api/v1/devices/"+url.PathEscape(created.Slug), nil, &got); err != nil {
+		return seeded{}, err
+	}
+	if len(got.Interfaces) == 0 {
+		return seeded{}, errors.New("discovery found no interfaces")
+	}
+	s.ifaces = got.Interfaces
+	return s, nil
+}
+
+// backfillAll waits for each device's first live samples, then writes history
+// for every graphable interface in parallel.
+func (c *seedConfig) backfillAll(ctx context.Context, devs []seeded) error {
+	type job struct{ dev, iface string }
+	var jobs []job
+	probe := &backfillConfig{hotURL: c.hotURL, hc: c.hc}
+	for _, d := range devs {
+		for _, i := range d.ifaces {
+			if d.hc && i.graphable() {
+				jobs = append(jobs, job{d.slug, i.ID})
+			}
+		}
+		if err := c.waitLive(ctx, probe, d); err != nil {
+			return err
+		}
+		if d.kind == "down" {
+			_ = d.agent.Close() // goes silent now, so it reads as down by the time the screenshots are taken
+		}
+	}
+	ch := make(chan job)
+	errs := make(chan error, len(jobs))
+	var wg sync.WaitGroup
+	for w := 0; w < max(1, c.workers); w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := range ch {
+				cfg := &backfillConfig{hotURL: c.hotURL, device: j.dev, days: c.days, step: c.step, pattern: patternDiurnal,
+					seed: 1, hc: c.hc, logf: func(string, ...any) {}}
+				errs <- cfg.backfillIface(ctx, j.iface)
+			}
+		}()
+	}
+	for _, j := range jobs {
+		ch <- j
+	}
+	close(ch)
+	wg.Wait()
+	close(errs)
+	for err := range errs {
+		if err != nil {
+			return err
+		}
+	}
+	fmt.Printf("back-filled %d interfaces, %d days at %s\n", len(jobs), c.days, c.step)
+	return nil
+}
+
+// waitLive blocks until the hot store holds two live samples of the device's
+// first graphable interface, so the back-fill can end where they begin.
+func (c *seedConfig) waitLive(ctx context.Context, probe *backfillConfig, d seeded) error {
+	if !d.hc {
+		return nil
+	}
+	var first string
+	for _, i := range d.ifaces {
+		if i.graphable() {
+			first = i.ID
+			break
+		}
+	}
+	if first == "" {
+		return nil
+	}
+	deadline := time.Now().Add(3 * time.Minute)
+	for time.Now().Before(deadline) {
+		live, err := probe.probe(ctx, "ifHCInOctets", first)
+		if err == nil && live.ok && live.rate > 0 {
+			return nil
+		}
+		if !sleepCtx(ctx, 2*time.Second) {
+			return ctx.Err()
+		}
+	}
+	return fmt.Errorf("%s: no live samples in the hot store after 3 minutes (is nmon polling? --poll-interval)", d.slug)
+}
+
+func (c *seedConfig) writeEnv(devs []seeded) error {
+	var b bytes.Buffer
+	for _, d := range devs {
+		switch d.kind {
+		case "counters":
+			fmt.Fprintf(&b, "NMON_DEVICE=%s\n", d.slug)
+		case "big":
+			fmt.Fprintf(&b, "NMON_DEVICE_48=%s\n", d.slug)
+		case "v1":
+			fmt.Fprintf(&b, "NMON_DEVICE_32BIT=%s\n", d.slug)
+		case "down":
+			fmt.Fprintf(&b, "NMON_DEVICE_DOWN=%s\n", d.slug)
+		}
+	}
+	if c.envOut == "" {
+		return nil
+	}
+	return os.WriteFile(c.envOut, b.Bytes(), 0o600)
+}

cmd/nmon-devtools/store.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/v10networks/nmon/internal/metrics"
+)
+
+// parseExport reads VictoriaMetrics' /api/v1/export (JSON lines) and returns
+// the earliest sample of the first series.
+func parseExport(raw []byte) (liveSeries, error) {
+	line, _, _ := bytes.Cut(bytes.TrimSpace(raw), []byte("\n"))
+	if len(line) == 0 {
+		return liveSeries{}, nil
+	}
+	var s struct {
+		Values     []float64 `json:"values"`
+		Timestamps []int64   `json:"timestamps"`
+	}
+	if err := json.Unmarshal(line, &s); err != nil {
+		return liveSeries{}, fmt.Errorf("decode export: %w", err)
+	}
+	if len(s.Values) == 0 || len(s.Values) != len(s.Timestamps) {
+		return liveSeries{}, nil
+	}
+	out := liveSeries{ts: time.UnixMilli(s.Timestamps[0]).UTC(), val: s.Values[0], ok: true}
+	if len(s.Values) > 1 && s.Timestamps[1] > s.Timestamps[0] {
+		out.rate = (s.Values[1] - s.Values[0]) / (float64(s.Timestamps[1]-s.Timestamps[0]) / 1000)
+	}
+	return out, nil
+}
+
+func (c *backfillConfig) get(ctx context.Context, u string, dst io.Writer) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := c.hc.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }() // read to the end below
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
+		return fmt.Errorf("GET %s: %s: %s", u, resp.Status, b)
+	}
+	_, err = io.Copy(dst, resp.Body)
+	return err
+}
+
+// importBatch is how many samples go in one POST (about 5 MB of text).
+const importBatch = 50_000
+
+// send writes samples through /api/v1/import/prometheus in batches.
+func (c *backfillConfig) send(ctx context.Context, samples []metrics.Sample) error {
+	for len(samples) > 0 {
+		n := min(len(samples), importBatch)
+		body := make([]byte, 0, n*110)
+		for _, s := range samples[:n] {
+			body = metrics.AppendPromText(body, s)
+		}
+		req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.hotURL+"/api/v1/import/prometheus", bytes.NewReader(body))
+		if err != nil {
+			return err
+		}
+		resp, err := c.hc.Do(req)
+		if err != nil {
+			return err
+		}
+		msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
+		_ = resp.Body.Close() // reply already read
+		if resp.StatusCode/100 != 2 {
+			return fmt.Errorf("import: %s: %s", resp.Status, msg)
+		}
+		samples = samples[n:]
+	}
+	return nil
+}

cmd/nmon/openapi.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/v10networks/nmon/internal/app"
+)
+
+func init() {
+	register("openapi", "print the generated OpenAPI 3.1 document (nmon openapi > openapi.json)", openapiCmd)
+}
+
+func openapiCmd(_ context.Context, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("openapi takes no arguments, got %q", args[0])
+	}
+	spec, err := app.OpenAPISpec()
+	if err != nil {
+		return fmt.Errorf("generate spec: %w", err)
+	}
+	_, err = fmt.Fprintf(os.Stdout, "%s\n", spec)
+	return err
+}

docs/DEVIATIONS.md

@@ -14,3 +14,10 @@ conscious, reviewed exception.
 | 7 | `import-librenms` reads `includes/definitions/*.yaml` and `includes/definitions/discovery/*.yaml` | Reads `resources/definitions/os_detection/*.yaml` and `resources/definitions/os_discovery/*.yaml`, with the old paths as a fallback. | LibreNMS moved them; the spec's paths no longer exist upstream. | M2 |
 | 8 | Real recordings replace synthetic fixtures when available | LibreNMS's `tests/snmpsim/*.snmprec` (2,054 real device recordings) are used by optional tests when `NMON_LIBRENMS_DIR` points at a checkout. They are never copied into this repo. | Real-device coverage without vendoring GPL test data into an Apache-2.0 repo. | M2 |
 | 9 | `nmon profile import-librenms <repo-path>` is an nmon subcommand | A separate binary, `nmon-librenms-import` (`cmd/nmon-librenms-import`), run offline before nmon. It writes draft profiles that the operator reviews and copies into `<data-dir>/profiles/`. `nmon` neither links nor exposes it. | User decision (2026-10-22). It keeps the importer out of the product binary and makes "the converter never runs at serve time" structural. | M2 |
+| 10 | `scripts/openapi-diff` fails on an incompatible change (M3 milestone) | The differ is the Go package `internal/openapidiff` (unit-tested, run with `go run ./internal/openapidiff/cmd/openapidiff`), driven by `scripts/openapi-check.sh`, which `make gate-m3` and the CI `openapi` job call. | A Go package is testable and linted like the rest of the code; the script only fetches the spec at the last `m*`/`v*` tag. No new dependency. | M3 |
+| 11 | Built assets are embedded from `web/dist` into `internal/webui` | `make web` copies `web/dist` to `internal/webui/dist` (git-ignored, `.gitkeep` tracked); a committed `internal/webui/placeholder/index.html` ("UI not built") is served when `dist` has no `index.html`. | A tracked `dist/index.html` would be overwritten by every build and dirty the tree, and a built UI must never be committed. `go build` still works without node. | M3 |
+| 12 | (not specified) | Developer tools (`backfill`, `seed-demo`) are a separate executable, `cmd/nmon-devtools` (`make devtools` builds `nmon-devtools`), never shipped. It talks to a running nmon over its HTTP API and to the hot store via `--hot-url`. `nmon` has no `devtools` command; a depguard rule keeps `cmd/nmon` from importing `internal/simagent` or `internal/fixturegen` beyond `profile_record.go`, which already used them for `nmon profile record` in M2. | The gate needs 30 days of history without waiting 30 days, and live screenshots need a simulated network. See `docs/dev.md`. | M3 |
+| 13 | (not specified) `/docs` API reference | `/docs` serves a vendored, pinned swagger-ui-dist 5.17.14 (Apache-2.0, about 1.6 MB) through `embed.FS`. | huma's built-in renderers load their assets from a CDN, which an offline monitor cannot rely on. | M3 |
+| 14 | M2 defined provisional `poller/health` and `identity-reviews` responses | Both were reshaped in M3 to the reviewed UI contract, before any `openapi.json` existed, so the route freeze starts at `m3`, the first tag with a checked-in spec. | The M2 shapes had no consumer. `scripts/openapi-check.sh` passes against `m2` with a note and diffs for real from `m3`. | M3 |
+| 15 | A single-signal failure (ICMP only or SNMP only) is a warning | The warning is held as pending for 5 min before it notifies, and is closed silently if the device goes down meanwhile. | Otherwise a real outage whose second signal fails seconds later sends a spurious warning first. | M4 |
+| 16 | An unsupported probe is unknown, never failed | When ICMP is unsupported (host can't open unprivileged ICMP, or per device), SNMP alone decides down, as with `icmp_blocked`, and a one-time `icmp_unsupported` device warning is raised. | Otherwise nmon could never declare a device down on such a host, and nobody would notice. | M4 |

docs/dev.md

@@ -0,0 +1,50 @@
+# Developer notes
+
+## Building the UI into the binary
+
+`make build` runs `make web` first: `npm run build` in `web/`, then `web/dist` is copied to
+`internal/webui/dist`, which `//go:embed` packs into the binary. `internal/webui/dist` is git-ignored
+(only `.gitkeep` is tracked), so a build never dirties the tree. A plain `go build` without `make web`
+embeds a one-line "UI not built" page from `internal/webui/placeholder/`.
+
+nmon serves the UI at `/`. Unknown paths fall back to `index.html` for the client-side router, except
+`/api/`, `/docs`, `/openapi*`, `/metrics` and `/schemas`, which belong to the API. Files under `/assets/`
+are content-hashed and sent `immutable`; `index.html` is `no-cache`; text assets are served gzipped.
+
+## OpenAPI compatibility
+
+`scripts/openapi-check.sh` compares the current spec (`nmon openapi`, else `openapi.json`) with
+`openapi.json` at the latest `m*`/`v*` tag and fails on a removed route or method, a removed response
+field, a changed field type, a new required parameter or request field, or a removed enum value. A tag
+without `openapi.json` passes with a note. `OPENAPI_NEW=file.json` compares that file instead and
+`OPENAPI_BASE_TAG=m3` picks the base. The comparison is `internal/openapidiff`.
+
+## `nmon-devtools` (separate dev binary)
+
+Developer helpers in their own executable, `make devtools` -> `nmon-devtools`; the shipped `nmon` does not contain them and never links the simulator. They need a running `nmon serve --insecure-http`.
+
+**backfill** writes synthetic history straight into the hot store so graphs have 30 days of data without
+waiting 30 days:
+
+    nmon-devtools backfill --hot-url http://127.0.0.1:PORT --device SLUG --iface ULID [--iface ULID...] \
+        [--days 31] [--step 60s] [--pattern diurnal|flat] [--mean-mbps N]
+
+Per interface it writes `ifHCInOctets`, `ifHCOutOctets` (monotonic, diurnal rate with noise and bursts) and
+`ifOperStatus`, ending one step before the first live sample. When live samples exist the history is shifted
+so its last value joins the live counter with no reset (this needs a live counter larger than the history,
+about 3e13 bytes for 31 days at 100 Mbit/s; otherwise a warning says the join will look like a reset).
+The hot URL is in the serve log (`metric stores up hot=...`). Samples older than hot retention are dropped
+by the store.
+
+**seed-demo** builds a small simulated network and back-fills it, for the gate and for screenshots:
+
+    nmon serve --data-dir D --insecure-http --poll-interval 10s  ...
+    nmon-devtools seed-demo --api http://127.0.0.1:8443 --hot-url http://127.0.0.1:PORT \
+        [--env-out env] [--counters-rec testdata/snmprec/counters.snmprec]
+
+It starts the simulated SNMP agents in-process on `[::1]`, each on its own port (passed to the API as `snmp.port`), adds the devices through the API (a 48-port switch, access switches, routers, a firewall, a UPS, the counters fixture, a v1 device with 32-bit counters, and one whose agent is stopped after seeding so it reads as down), waits for live polls, back-fills 31 days, writes `NMON_DEVICE`, `NMON_DEVICE_48`, `NMON_DEVICE_32BIT` and `NMON_DEVICE_DOWN` to `--env-out`, and keeps the agents answering until interrupted. The simulated 64-bit counters start at 1e14 so history fits below them.
+
+## Gates
+
+`make gate-m3` runs everything above end to end and writes live screenshots to `docs/screenshots/M3/`.
+It needs Playwright's chromium: `cd web && npx playwright install chromium`.

docs/milestones/M3-graphs-and-ui.md

@@ -33,7 +33,20 @@ API and UI: openapi.json generated with the TS client and the incompatible-diff
 
 ## Gate log
 
-_(filled in by the supervisor when the gate passes)_
+**Passed 2026-10-22**, run by the supervisor (Opus).
+
+| Check | Result |
+|---|---|
+| `make lint` (Go + web) | 0 findings |
+| `go test -race ./...`, `-tags 'integration vmembed'`, `test-pg.sh` | all ok |
+| `make gate-m1`, `make gate-m2` (with LibreNMS recordings) | PASS |
+| `make gate-m3` | PASS: UI embedded in the static binary; SPA fallback and cache headers; 10 simulated devices (`nmon-devtools seed-demo`) with 31 days of back-filled history; port graph renders in and out at 1 h, 24 h, 30 d; 48-port page < 2 s; openapi-check; web client regenerated from the live spec type-checks |
+| UI ↔ API contract | `TestGeneratedSpecMatchesUIContract`: the generated `openapi.json` is compatible with the contract the UI was built against |
+| Screenshots | `docs/screenshots/M3/` (real data, 18 files) and `docs/screenshots/M3-mock/` (mock data); reviewed by the supervisor and sent to the owner |
+
+**Decisions in M3:** dev tooling is a separate `nmon-devtools` binary (owner's call), never shipped. `/docs` uses vendored Swagger UI. The two M2 provisional routes were reshaped before the route freeze, which starts at `m3`, the first tag with `openapi.json`. Graph stitching joins at now − hot retention; p95 is computed live until M5 materialises it.
+
+**Carried forward:** a demo device that is genuinely Down needs an unreachable address; that comes with M4's reachability work (`device-unresponsive.png` shows Warning/degraded for now, which is correct per policy because ICMP still answers). `nmon profile record` still links the simulator package; to be moved to a shared `internal/snmprec` format package.
 
 ## API contract for M3 (written before the WPs so the UI and API are built in parallel)
 

docs/milestones/README.md

@@ -14,8 +14,8 @@ its **Gate log** is filled in when the gate passes. A milestone is done only whe
 |---|---|---|---|---|
 | M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | **passed**, tagged `m1` |
 | M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | **passed**, tagged `m2` |
-| M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | in progress (web UI done on mock data) |
-| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | in progress (WP2 rules and WP3 notify done) |
+| M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | **passed**, tagged `m3` |
+| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | in progress (WP1 reach and deps, WP2 rules, WP3 notify done) |
 | M5 | Push ingest, rollup, checks | a replayed Proxmox sample shows under its node and a 2 y query hits the 1 h tier | [M5](M5-ingest-rollup-checks.md) | planned |
 | M6 | Auth and ops | a viewer gets 403 on PUT and the 2,000-device loadtest cycle finishes under 40 s | [M6](M6-auth-and-ops.md) | planned |
 | M7 | gNMI streaming | after v1; not built in this run | — | out of scope |

internal/api/admit.go

@@ -0,0 +1,99 @@
+package api
+
+import (
+	"context"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+)
+
+// admission keeps metric-store query load bounded: a per-client cap on
+// free-form queries (so one dashboard cannot starve the rest) and a global
+// semaphore on every store query (so the UI cannot crowd out the poll path).
+// A request over its client cap is refused at once; one waiting for a global
+// slot waits up to wait, then is refused. Both are 429 problems.
+type admission struct {
+	global    chan struct{}
+	perClient int
+	wait      time.Duration
+
+	mu     sync.Mutex
+	active map[string]int
+}
+
+func newAdmission(perClient, global int, wait time.Duration) *admission {
+	return &admission{global: make(chan struct{}, global), perClient: perClient, wait: wait, active: map[string]int{}}
+}
+
+// clientKey carries the caller's identity from the middleware to the handlers.
+// Until authentication exists (M6) it is the remote address without the port.
+type clientKey struct{}
+
+func clientKeyMiddleware(ctx huma.Context, next func(huma.Context)) {
+	host, _, err := net.SplitHostPort(ctx.RemoteAddr())
+	if err != nil {
+		host = ctx.RemoteAddr()
+	}
+	next(huma.WithValue(ctx, clientKey{}, host))
+}
+
+func clientOf(ctx context.Context) string {
+	c, _ := ctx.Value(clientKey{}).(string)
+	return c
+}
+
+func tooMany(msg string) error {
+	return huma.Error429TooManyRequests(msg)
+}
+
+// acquire admits one query. freeForm queries also count against the client's cap.
+// The returned release must be called exactly once.
+func (a *admission) acquire(ctx context.Context, freeForm bool) (release func(), err error) {
+	client := clientOf(ctx)
+	if freeForm && !a.takeClient(client) {
+		return nil, tooMany("too many concurrent queries from " + client + "; the limit is " + itoa(a.perClient))
+	}
+	wctx, cancel := context.WithTimeout(ctx, a.wait)
+	defer cancel()
+	select {
+	case a.global <- struct{}{}:
+	case <-wctx.Done():
+		if freeForm {
+			a.dropClient(client)
+		}
+		return nil, tooMany("the metric store is busy; try again shortly")
+	}
+	return func() {
+		<-a.global
+		if freeForm {
+			a.dropClient(client)
+		}
+	}, nil
+}
+
+func (a *admission) takeClient(client string) bool {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if a.active[client] >= a.perClient {
+		return false
+	}
+	a.active[client]++
+	return true
+}
+
+func (a *admission) dropClient(client string) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if a.active[client] <= 1 {
+		delete(a.active, client)
+		return
+	}
+	a.active[client]--
+}
+
+// admit takes a query slot for the request in ctx.
+func (s *server) admit(ctx context.Context, freeForm bool) (func(), error) {
+	return s.adm.acquire(ctx, freeForm)
+}

internal/api/api.go

@@ -3,6 +3,7 @@
 package api
 
 import (
+	"cmp"
 	"context"
 	"encoding/json"
 	"errors"
@@ -32,6 +33,8 @@ const (
 	MaxSeries = 100_000
 	// DiscoveryTimeout bounds the seed discovery run by POST /devices.
 	DiscoveryTimeout = 30 * time.Second
+	// Version is the API's OpenAPI info.version; it changes only with the route contract, never with a build.
+	Version = "1.0.0"
 )
 
 // Querier runs read-only MetricsQL against the hot store. *vmsup.Querier implements it.
@@ -62,14 +65,33 @@ type Deps struct {
 
 	// DiscoveryTimeout overrides the 30 s seed discovery limit (tests).
 	DiscoveryTimeout time.Duration
+
+	// Reachability decides up/down/warning per device. Default: what is persisted now
+	// (poll state, last_seen, the ICMP and SNMP up series). M4 swaps in the alerter's state.
+	Reachability ReachabilitySource
+	// PollInterval is the normal poll cadence; a device unheard of for three of them is down. Default 60 s.
+	PollInterval time.Duration
+
+	// HotRetention, FiveMinuteRetention and LongRetention are the stores' retention periods. They decide where a
+	// graph stitches and what GET /meta reports. Defaults: 30 d, 120 d, 2 y.
+	HotRetention, FiveMinuteRetention, LongRetention time.Duration
+	// WriteQueueCapacity is the hot write queue's size in samples, for GET /poller/health. Default 2,000,000.
+	WriteQueueCapacity int
+
+	// MaxClientQueries and MaxGlobalQueries bound concurrent free-form queries per client and metric-store
+	// queries overall (defaults 4 and 16). AdmissionWait is how long a request waits for a global slot (default 5 s).
+	MaxClientQueries, MaxGlobalQueries int
+	AdmissionWait                      time.Duration
 }
 
 type server struct {
 	Deps
+	adm  *admission
+	rate *counterRates
 }
 
-// New returns the API handler with every route registered.
-func New(d Deps) http.Handler {
+// withDefaults fills every unset optional dependency.
+func (d Deps) withDefaults() Deps {
 	if d.Clock == nil {
 		d.Clock = clock.Real()
 	}
@@ -79,25 +101,72 @@ func New(d Deps) http.Handler {
 	if d.Logger == nil {
 		d.Logger = slog.Default()
 	}
-	if d.DiscoveryTimeout <= 0 {
-		d.DiscoveryTimeout = DiscoveryTimeout
+	d.DiscoveryTimeout = cmp.Or(d.DiscoveryTimeout, DiscoveryTimeout)
+	d.PollInterval = cmp.Or(d.PollInterval, time.Minute)
+	d.HotRetention = cmp.Or(d.HotRetention, 30*day)
+	d.FiveMinuteRetention = cmp.Or(d.FiveMinuteRetention, 120*day)
+	d.LongRetention = cmp.Or(d.LongRetention, 2*365*day)
+	d.WriteQueueCapacity = cmp.Or(d.WriteQueueCapacity, 2_000_000)
+	d.MaxClientQueries = cmp.Or(d.MaxClientQueries, 4)
+	d.MaxGlobalQueries = cmp.Or(d.MaxGlobalQueries, 16)
+	d.AdmissionWait = cmp.Or(d.AdmissionWait, 5*time.Second)
+	if d.Reachability == nil {
+		d.Reachability = &persistedReachability{q: d.Querier, clk: d.Clock, interval: d.PollInterval}
 	}
-	s := &server{d}
+	return d
+}
 
+const day = 24 * time.Hour
+
+// New returns the API handler with every route registered.
+func New(d Deps) http.Handler {
+	d = d.withDefaults()
+	s := &server{Deps: d, adm: newAdmission(d.MaxClientQueries, d.MaxGlobalQueries, d.AdmissionWait), rate: &counterRates{}}
+	s.rate.perMinute(s.readCounters()) // baseline, so the first poller/health call has a window to measure
 	mux := http.NewServeMux()
-	cfg := huma.DefaultConfig("nmon", "1.0.0")
+	api := s.newHuma(mux)
+	registerDocs(mux)
+	s.register(api)
+	return problemFallback(mux)
+}
+
+// Spec returns the OpenAPI 3.1 document of the API as indented JSON. It is the
+// same document served at /openapi.json, generated without any dependencies:
+// `nmon openapi > openapi.json` checks it in.
+func Spec() ([]byte, error) {
+	s := &server{Deps: Deps{}.withDefaults()}
+	api := s.newHuma(http.NewServeMux())
+	s.register(api)
+	return json.MarshalIndent(api.OpenAPI(), "", "  ")
+}
+
+// register adds every operation to api.
+func (s *server) register(api huma.API) {
+	s.registerDevices(api)
+	s.registerDeviceList(api)
+	s.registerDeviceViews(api)
+	s.registerInterfaces(api)
+	s.registerGraph(api)
+	s.registerOps(api)
+	s.registerHealth(api)
+	s.registerReviews(api)
+	s.registerQuery(api)
+	s.registerMeta(api)
+	s.registerMetrics(api)
+}
+
+// newHuma builds the huma API on mux. /docs is served by registerDocs from embedded assets, not huma's CDN page.
+func (s *server) newHuma(mux *http.ServeMux) huma.API {
+	configureHuma()
+	cfg := huma.DefaultConfig("nmon", Version)
 	cfg.Info.Description = "nmon network monitor API."
 	cfg.OpenAPIPath = "/openapi"
-	cfg.DocsPath = "/docs"
-	cfg.DocsRenderer = huma.DocsRendererSwaggerUI
+	cfg.DocsPath = ""
 	cfg.CreateHooks = nil // no $schema fields or Link headers in responses
+	cfg.OnAddOperation = append(cfg.OnAddOperation, addDefaultProblem)
 	api := humago.New(mux, cfg)
-
-	s.registerDevices(api)
-	s.registerOps(api)
-	s.registerQuery(api)
-	s.registerMetrics(api)
-	return problemFallback(mux)
+	api.UseMiddleware(clientKeyMiddleware)
+	return api
 }
 
 // storeErr maps store sentinels to problem responses; anything else is a 500

internal/api/api_test.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -25,25 +26,57 @@ import (
 	"github.com/v10networks/nmon/internal/vmsup"
 )
 
+// querier is the fake metric store. fn, when set, answers every query;
+// otherwise body does. Every query is recorded.
 type querier struct {
+	mu                sync.Mutex
 	body              []byte
 	err               error
+	fn                func(store, expr string, from, to time.Time, step time.Duration) ([]byte, error)
 	q                 string
 	ts, from, to      time.Time
 	step              time.Duration
 	instant, rangeCnt int
+	calls             []call
+}
+
+type call struct {
+	store, expr string
+	from, to    time.Time
+	step        time.Duration
 }
 
 func (q *querier) Query(_ context.Context, st, expr string, ts time.Time) ([]byte, error) {
+	q.mu.Lock()
 	q.instant++
 	q.q, q.ts = expr, ts
-	return q.body, q.err
+	q.calls = append(q.calls, call{store: st, expr: expr, from: ts, to: ts})
+	fn, body, err := q.fn, q.body, q.err
+	q.mu.Unlock()
+	if fn != nil {
+		return fn(st, expr, ts, ts, 0)
+	}
+	return body, err
 }
 
 func (q *querier) QueryRange(_ context.Context, st, expr string, from, to time.Time, step time.Duration) ([]byte, error) {
+	q.mu.Lock()
 	q.rangeCnt++
 	q.q, q.from, q.to, q.step = expr, from, to, step
-	return q.body, q.err
+	q.calls = append(q.calls, call{store: st, expr: expr, from: from, to: to, step: step})
+	fn, body, err := q.fn, q.body, q.err
+	q.mu.Unlock()
+	if fn != nil {
+		return fn(st, expr, from, to, step)
+	}
+	return body, err
+}
+
+// queries returns a copy of the recorded queries.
+func (q *querier) queries() []call {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return append([]call(nil), q.calls...)
 }
 
 type refresher struct{ n atomic.Int32 }
@@ -51,6 +84,7 @@ type refresher struct{ n atomic.Int32 }
 func (r *refresher) Refresh() { r.n.Add(1) }
 
 type env struct {
+	clk *clock.Fake
 	h   http.Handler
 	st  store.Store
 	q   *querier
@@ -61,14 +95,21 @@ type env struct {
 	status  []poller.DeviceStatus
 }
 
-func newEnv(t *testing.T, port int) *env {
+func newEnv(t *testing.T, port int) *env { return newEnvDeps(t, port, nil) }
+
+// newEnvDeps is newEnv with a chance to change the API's dependencies.
+func newEnvDeps(t *testing.T, port int, mutate func(*api.Deps)) *env {
 	clk := clock.NewFake(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC))
-	e := &env{st: testutil.OpenStore(t, clk), q: &querier{body: []byte(`{"status":"success","data":{"resultType":"vector","result":[]}}`)},
+	e := &env{clk: clk, st: testutil.OpenStore(t, clk), q: &querier{body: []byte(`{"status":"success","data":{"resultType":"vector","result":[]}}`)},
 		ref: &refresher{}, reg: selfmon.New()}
-	e.h = api.New(api.Deps{Store: e.st, Querier: e.q, Poller: e.ref, Clock: clk, Registry: e.reg, SNMPPort: port,
+	d := api.Deps{Store: e.st, Querier: e.q, Poller: e.ref, Clock: clk, Registry: e.reg, SNMPPort: port,
 		DiscoveryTimeout: 3 * time.Second,
 		PollerStatus:     func() []poller.DeviceStatus { return e.status },
-		AfterDiscover:    func(id int64) { e.afterID.Store(id) }})
+		AfterDiscover:    func(id int64) { e.afterID.Store(id) }}
+	if mutate != nil {
+		mutate(&d)
+	}
+	e.h = api.New(d)
 	return e
 }
 
@@ -276,3 +317,24 @@ func TestQueryProxy(t *testing.T) {
 	e.q.err = errors.New("connection reset")
 	isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 502)
 }
+
+func TestDocsAreSelfHosted(t *testing.T) {
+	e := newEnv(t, 0)
+	page := e.do("GET", "/docs", "")
+	if page.Code != 200 || strings.Contains(page.Body.String(), "http") && strings.Contains(page.Body.String(), "unpkg") ||
+		!strings.Contains(page.Body.String(), "/docs/assets/swagger-ui-bundle.js") {
+		t.Fatalf("/docs: %d %s", page.Code, page.Body)
+	}
+	for path, ct := range map[string]string{"/docs/assets/swagger-ui-bundle.js": "javascript", "/docs/assets/swagger-ui.css": "css", "/docs/assets/init.js": "javascript"} {
+		rec := e.do("GET", path, "")
+		if rec.Code != 200 || rec.Body.Len() < 50 || !strings.Contains(rec.Header().Get("Content-Type"), ct) {
+			t.Errorf("%s: %d %s %d bytes", path, rec.Code, rec.Header().Get("Content-Type"), rec.Body.Len())
+		}
+	}
+	if !strings.Contains(e.do("GET", "/docs/assets/init.js", "").Body.String(), "/openapi.json") {
+		t.Error("the page must load the spec from this server")
+	}
+	if rec := e.do("GET", "/openapi.json", ""); rec.Code != 200 || !strings.Contains(rec.Body.String(), `"openapi":"3.1.0"`) {
+		t.Errorf("openapi.json: %d", rec.Code)
+	}
+}

internal/api/contract_test.go

@@ -0,0 +1,435 @@
+package api_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"slices"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/api"
+)
+
+// The UI is built against web/openapi.contract.json. These tests hold the
+// generated OpenAPI document to it: every path, method, parameter, request
+// body and response field of the contract must exist in the generated spec
+// with a compatible type. Additions in the generated spec are fine.
+
+type obj = map[string]any
+
+// knownDivergences are places where the generated spec deliberately differs
+// from the hand-written contract. Each has a reason; anything else is a failure.
+var knownDivergences = map[string]string{
+	// M1/M2 shipped these two as strings that also accept Unix seconds and "30s"; the routes are frozen once shipped
+	// and the UI does not call them.
+	"param GET /api/v1/query_range step: type":   "frozen since M1: accepts 30s and plain seconds",
+	"param GET /api/v1/query_range from: format": "frozen since M1: accepts RFC 3339 and Unix seconds",
+	"param GET /api/v1/query_range to: format":   "frozen since M1: accepts RFC 3339 and Unix seconds",
+	"param GET /api/v1/query time: format":       "frozen since M1: accepts RFC 3339 and Unix seconds",
+	// The contract left q optional; an empty expression is meaningless, so the API requires it.
+	"param GET /api/v1/query q: required":          "q is required by the frozen M1 route",
+	"param GET /api/v1/query_range q: required":    "q is required by the frozen M1 route",
+	"param GET /api/v1/query_range step: required": "step is required by the frozen M1 route",
+}
+
+type checker struct {
+	t      *testing.T
+	gen    obj
+	con    obj
+	failed []string
+}
+
+func (c *checker) fail(key, format string, a ...any) {
+	if _, ok := knownDivergences[key]; ok {
+		return
+	}
+	c.failed = append(c.failed, fmt.Sprintf("%s: %s", key, fmt.Sprintf(format, a...)))
+}
+
+func load(t *testing.T) *checker {
+	t.Helper()
+	raw, err := os.ReadFile("../../web/openapi.contract.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var con obj
+	if err := json.Unmarshal(raw, &con); err != nil {
+		t.Fatal(err)
+	}
+	spec, err := api.Spec()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var gen obj
+	if err := json.Unmarshal(spec, &gen); err != nil {
+		t.Fatal(err)
+	}
+	return &checker{t: t, gen: gen, con: con}
+}
+
+func get(m any, keys ...string) any {
+	for _, k := range keys {
+		mm, ok := m.(obj)
+		if !ok {
+			return nil
+		}
+		m = mm[k]
+	}
+	return m
+}
+
+// resolve follows $ref within doc.
+func resolve(doc obj, s any) obj {
+	m, _ := s.(obj)
+	for m != nil {
+		ref, ok := m["$ref"].(string)
+		if !ok {
+			return m
+		}
+		m, _ = get(doc, strings.Split(strings.TrimPrefix(ref, "#/"), "/")...).(obj)
+	}
+	return nil
+}
+
+// shape is a schema reduced to what the comparison needs.
+type shape struct {
+	typ      string
+	nullable bool
+	s        obj
+}
+
+func shapeOf(doc obj, raw any) shape {
+	s := resolve(doc, raw)
+	sh := shape{s: s}
+	switch t := s["type"].(type) {
+	case string:
+		sh.typ = t
+	case []any:
+		for _, x := range t {
+			if x == "null" {
+				sh.nullable = true
+			} else {
+				sh.typ, _ = x.(string)
+			}
+		}
+	}
+	if alts, ok := s["anyOf"].([]any); ok {
+		for _, a := range alts {
+			am := resolve(doc, a)
+			if am["type"] == "null" {
+				sh.nullable = true
+				continue
+			}
+			sub := shapeOf(doc, am)
+			sh.typ, sh.s = sub.typ, sub.s
+		}
+	}
+	return sh
+}
+
+// compat checks that gen can stand where con is expected. request flips the
+// direction of the required/nullable rules: a client sends what the contract
+// allows, and receives what the contract promises.
+func (c *checker) compat(key string, con, gen any, request bool) {
+	cs, gs := shapeOf(c.con, con), shapeOf(c.gen, gen)
+	if cs.typ != gs.typ && (cs.typ != "number" || gs.typ != "integer") {
+		c.fail(key+": type", "contract %q, generated %q", cs.typ, gs.typ)
+		return
+	}
+	if !request && cs.nullable && !gs.nullable {
+		c.fail(key+": nullable", "contract allows null, generated never sends it (fine) -- but UI types expect null; declare nullable")
+	}
+	if !request && gs.nullable && !cs.nullable {
+		c.fail(key+": nullable", "generated may send null, contract does not allow it")
+	}
+	for _, k := range []string{"format", "minimum", "maximum", "minLength", "maxLength", "minItems", "maxItems"} {
+		if cv, ok := cs.s[k]; ok && fmt.Sprint(cv) != fmt.Sprint(gs.s[k]) {
+			c.fail(key+": "+k, "contract %v, generated %v", cv, gs.s[k])
+		}
+	}
+	c.enums(key, cs.s, gs.s, request)
+	switch cs.typ {
+	case "object":
+		c.object(key, cs.s, gs.s, request)
+	case "array":
+		c.array(key, cs.s, gs.s, request)
+	}
+}
+
+func (c *checker) enums(key string, con, gen obj, request bool) {
+	ce, _ := con["enum"].([]any)
+	if len(ce) == 0 {
+		return
+	}
+	ge, _ := gen["enum"].([]any)
+	have := func(list []any, v any) bool { return slices.Contains(list, v) }
+	for _, v := range ge {
+		if !request && !have(ce, v) {
+			c.fail(key+": enum", "generated may send %v, not in the contract", v)
+		}
+	}
+	for _, v := range ce {
+		if (request || len(ge) == 0) && !have(ge, v) {
+			c.fail(key+": enum", "contract value %v missing from generated", v)
+		}
+	}
+}
+
+func (c *checker) object(key string, con, gen obj, request bool) {
+	cp, _ := con["properties"].(obj)
+	gp, _ := gen["properties"].(obj)
+	names := make([]string, 0, len(cp))
+	for n := range cp {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, n := range names {
+		if _, ok := gp[n]; !ok {
+			c.fail(key+"."+n, "missing from generated")
+			continue
+		}
+		c.compat(key+"."+n, cp[n], gp[n], request)
+	}
+	creq, greq := strset(con["required"]), strset(gen["required"])
+	if request {
+		for n := range greq {
+			if !creq[n] {
+				c.fail(key+"."+n+": required", "generated requires a field the contract made optional")
+			}
+		}
+		return
+	}
+	for n := range creq {
+		if !greq[n] {
+			c.fail(key+"."+n+": required", "contract promises it, generated may omit it")
+		}
+	}
+}
+
+func strset(v any) map[string]bool {
+	out := map[string]bool{}
+	l, _ := v.([]any)
+	for _, x := range l {
+		out[fmt.Sprint(x)] = true
+	}
+	return out
+}
+
+func (c *checker) array(key string, con, gen obj, request bool) {
+	if pc, ok := con["prefixItems"].([]any); ok {
+		pg, _ := gen["prefixItems"].([]any)
+		if len(pg) != len(pc) {
+			c.fail(key+": prefixItems", "contract has %d, generated %d", len(pc), len(pg))
+			return
+		}
+		for i := range pc {
+			c.compat(fmt.Sprintf("%s[%d]", key, i), pc[i], pg[i], request)
+		}
+		return
+	}
+	if con["items"] != nil {
+		if gen["items"] == nil {
+			c.fail(key+": items", "generated array has no items schema")
+			return
+		}
+		c.compat(key+"[]", con["items"], gen["items"], request)
+	}
+}
+
+// operations lists "METHOD path" for every operation of doc, with path-level parameters merged in.
+func operations(doc obj) map[string]obj {
+	out := map[string]obj{}
+	paths, _ := doc["paths"].(obj)
+	for p, item := range paths {
+		im, _ := item.(obj)
+		for m, op := range im {
+			om, ok := op.(obj)
+			if !ok || m == "parameters" {
+				continue
+			}
+			merged := obj{}
+			for k, v := range om {
+				merged[k] = v
+			}
+			params, _ := im["parameters"].([]any)
+			merged["parameters"] = append(slices.Clone(params), asList(om["parameters"])...)
+			out[strings.ToUpper(m)+" "+p] = merged
+		}
+	}
+	return out
+}
+
+func asList(v any) []any { l, _ := v.([]any); return l }
+
+func (c *checker) params(opKey string, con, gen obj) {
+	genByName := map[string]obj{}
+	for _, p := range asList(gen["parameters"]) {
+		pm := resolve(c.gen, p)
+		genByName[fmt.Sprint(pm["in"])+" "+fmt.Sprint(pm["name"])] = pm
+	}
+	conByName := map[string]bool{}
+	for _, p := range asList(con["parameters"]) {
+		pm := resolve(c.con, p)
+		id := fmt.Sprint(pm["in"]) + " " + fmt.Sprint(pm["name"])
+		conByName[id] = true
+		key := fmt.Sprintf("param %s %s", opKey, pm["name"])
+		g, ok := genByName[id]
+		if !ok {
+			c.fail(key, "missing from generated")
+			continue
+		}
+		if pm["required"] == true && g["required"] != true {
+			c.fail(key+": required", "contract requires it, generated does not")
+		}
+		if pm["required"] != true && g["required"] == true {
+			c.fail(key+": required", "generated requires a parameter the contract made optional")
+		}
+		c.compat(key, pm["schema"], g["schema"], true)
+	}
+	for id, g := range genByName {
+		if g["required"] == true && !conByName[id] {
+			c.fail(fmt.Sprintf("param %s %s", opKey, g["name"]), "new required parameter not in the contract")
+		}
+	}
+}
+
+func (c *checker) body(opKey string, con, gen obj) {
+	cb := resolve(c.con, get(con, "requestBody"))
+	if cb == nil {
+		return
+	}
+	gb := resolve(c.gen, get(gen, "requestBody"))
+	cs, gs := get(cb, "content", "application/json", "schema"), get(gb, "content", "application/json", "schema")
+	if gs == nil {
+		c.fail("body "+opKey, "generated has no JSON request body")
+		return
+	}
+	c.compat("body "+opKey, cs, gs, true)
+}
+
+func (c *checker) responses(opKey string, con, gen obj) {
+	cr, _ := con["responses"].(obj)
+	gr, _ := gen["responses"].(obj)
+	for code, r := range cr {
+		g, ok := gr[code]
+		if !ok {
+			c.fail("response "+opKey+" "+code, "missing from generated")
+			continue
+		}
+		cm, _ := get(resolve(c.con, r), "content").(obj)
+		for ct, media := range cm {
+			gm := get(resolve(c.gen, g), "content", ct, "schema")
+			if gm == nil {
+				c.fail("response "+opKey+" "+code, "no %s content in generated", ct)
+				continue
+			}
+			c.compat(fmt.Sprintf("response %s %s", opKey, code), get(media, "schema"), gm, false)
+		}
+	}
+}
+
+func TestGeneratedSpecMatchesUIContract(t *testing.T) {
+	c := load(t)
+	if c.gen["openapi"] != "3.1.0" {
+		t.Fatalf("generated openapi version %v, want 3.1.0", c.gen["openapi"])
+	}
+	conOps, genOps := operations(c.con), operations(c.gen)
+	for key, con := range conOps {
+		gen, ok := genOps[key]
+		if !ok {
+			c.fail("operation "+key, "missing from generated spec")
+			continue
+		}
+		c.params(key, con, gen)
+		c.body(key, con, gen)
+		c.responses(key, con, gen)
+	}
+	if len(c.failed) > 0 {
+		sort.Strings(c.failed)
+		t.Fatalf("generated spec breaks the UI contract (%d):\n  %s", len(c.failed), strings.Join(c.failed, "\n  "))
+	}
+}
+
+// TestContractDivergencesStayNeeded fails when a divergence is no longer needed, so the list shrinks.
+func TestContractDivergencesStayNeeded(t *testing.T) {
+	c := load(t)
+	saved := knownDivergences
+	defer func() { knownDivergences = saved }()
+	knownDivergences = map[string]string{}
+	conOps, genOps := operations(c.con), operations(c.gen)
+	for key, con := range conOps {
+		if gen, ok := genOps[key]; ok {
+			c.params(key, con, gen)
+		}
+	}
+	hit := strings.Join(c.failed, "\n")
+	for k := range saved {
+		if !strings.Contains(hit, k) {
+			t.Errorf("divergence %q no longer happens; remove it", k)
+		}
+	}
+}
+
+// TestResponseSchemasExposeNoIDsOrCredentials walks every schema reachable from a
+// response in the generated spec: no integer id, and no credential-like property.
+func TestResponseSchemasExposeNoIDsOrCredentials(t *testing.T) {
+	c := load(t)
+	seen := map[string]bool{}
+	var walk func(path string, raw any)
+	walk = func(path string, raw any) {
+		m, _ := raw.(obj)
+		if ref, ok := m["$ref"].(string); ok {
+			if seen[ref] {
+				return
+			}
+			seen[ref] = true
+			walk(path+"→"+ref, resolve(c.gen, m))
+			return
+		}
+		props, _ := m["properties"].(obj)
+		for name, p := range props {
+			lower := strings.ToLower(name)
+			if (lower == "id" || strings.HasSuffix(lower, "_id")) && shapeOf(c.gen, p).typ != "string" {
+				t.Errorf("%s.%s is not a string id", path, name)
+			}
+			for _, bad := range []string{"community", "password", "pass", "hash", "secret", "token", "credential", "key"} {
+				if strings.Contains(lower, bad) {
+					t.Errorf("%s.%s looks like a credential", path, name)
+				}
+			}
+			walk(path+"."+name, p)
+		}
+		if items := m["items"]; items != nil {
+			walk(path+"[]", items)
+		}
+		for _, k := range []string{"anyOf", "oneOf", "allOf"} {
+			for _, a := range asList(m[k]) {
+				walk(path, a)
+			}
+		}
+	}
+	for key, op := range operations(c.gen) {
+		responses, _ := op["responses"].(obj)
+		for code, r := range responses {
+			walk(key+" "+code, get(resolve(c.gen, r), "content", "application/json", "schema"))
+		}
+	}
+}
+
+// TestCheckedInOpenAPIIsCurrent keeps openapi.json (the file the UI client is
+// generated from) equal to what the handlers produce.
+func TestCheckedInOpenAPIIsCurrent(t *testing.T) {
+	want, err := api.Spec()
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := os.ReadFile("../../openapi.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.TrimSpace(string(got)) != strings.TrimSpace(string(want)) {
+		t.Fatal("openapi.json is stale: run `go run ./cmd/nmon openapi > openapi.json`")
+	}
+}

internal/api/detail_test.go

@@ -0,0 +1,226 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+func TestDeviceDetailLiveRates(t *testing.T) {
+	e := newEnv(t, 0)
+	core := e.device(t, "Core", "yyz", "core", func(d *store.Device) { d.HWSerial, d.SysDescr = "SN1", "JunOS" })
+	other := e.device(t, "Other", "", "")
+	i1 := e.iface(t, core, 1, "xe-0/0/1", "up")
+	i2 := e.iface(t, core, 2, "xe-0/0/2", "32bit", func(i *store.Interface) { i.HasHCCounters = false })
+	i3 := e.iface(t, core, 3, "xe-0/0/3", "bad", func(i *store.Interface) { i.Unreliable, i.HasHCCounters = true, false })
+	i4 := e.iface(t, core, 4, "xe-0/0/4", "idle")
+	i5 := e.iface(t, core, 5, "xe-0/0/5", "gone")
+	if err := e.st.RetireInterface(t.Context(), i5.ID); err != nil {
+		t.Fatal(err)
+	}
+	if err := e.st.SetParents(t.Context(), core.ID, store.RelNetwork, []int64{other.ID}); err != nil {
+		t.Fatal(err)
+	}
+	e.q.fn = func(_, expr string, _, _ time.Time, _ time.Duration) ([]byte, error) {
+		if !strings.Contains(expr, `device="`+core.Slug+`"`) {
+			return emptyMatrix, nil
+		}
+		l := func(id, k string) map[string]string { return map[string]string{"iface": id, "k": k} }
+		return vector(
+			vrow{l(i1.ID, "in_hc"), 8e6}, vrow{l(i1.ID, "out_hc"), 4e6}, vrow{l(i1.ID, "in_err"), 0.5}, vrow{l(i1.ID, "out_err"), 0},
+			vrow{l(i1.ID, "in_32"), 999}, // a stale 32-bit series must not win over the 64-bit counter
+			vrow{l(i2.ID, "in_32"), 100}, vrow{l(i2.ID, "out_32"), 200},
+			vrow{l(i3.ID, "in_32"), 5}, vrow{l(i3.ID, "out_32"), 5}, vrow{l(i3.ID, "in_err"), 1},
+			vrow{map[string]string{"k": "uptime"}, 86400},
+		), nil
+	}
+	var d api.DeviceView
+	rec := e.getJSON(t, "/api/v1/devices/"+core.Slug, &d)
+	assertNoLeaks(t, "device detail", rec.Body.Bytes())
+	var rateQueries []string
+	for _, c := range e.q.queries() {
+		if strings.Contains(c.expr, "ifHCInOctets") {
+			rateQueries = append(rateQueries, c.expr)
+		}
+	}
+	if len(rateQueries) != 1 || !strings.Contains(rateQueries[0], "rate(ifHCInOctets{device=") || !strings.Contains(rateQueries[0], "[5m])*8") {
+		t.Errorf("want one batched rate query, got %q", rateQueries)
+	}
+	if d.Slug != core.Slug || d.HWSerial != "SN1" || d.SysDescr != "JunOS" || d.UptimeSeconds == nil || *d.UptimeSeconds != 86400 ||
+		len(d.Parents) != 1 || d.Parents[0].Slug != other.Slug || d.Parents[0].Name != "Other" || d.Groups == nil ||
+		d.Reachability == "" || d.PollState != "ok" || len(d.Interfaces) != 5 {
+		t.Fatalf("detail: %+v", d)
+	}
+	by := map[string]api.InterfaceView{}
+	for _, i := range d.Interfaces {
+		by[i.ID] = i
+	}
+	if x := by[i1.ID]; x.InBPS == nil || *x.InBPS != 8e6 || *x.OutBPS != 4e6 || *x.InErrorsPS != 0.5 || x.OutErrorsPS == nil {
+		t.Errorf("hc interface: %+v", x)
+	}
+	if x := by[i2.ID]; x.InBPS == nil || *x.InBPS != 100 || *x.OutBPS != 200 || x.InErrorsPS != nil {
+		t.Errorf("32-bit interface: %+v", x)
+	}
+	if x := by[i3.ID]; !x.Unreliable || x.InBPS != nil || x.OutBPS != nil || x.InErrorsPS != nil {
+		t.Errorf("unreliable interface must have null rates: %+v", x)
+	}
+	if x := by[i4.ID]; x.InBPS != nil || x.OutBPS != nil {
+		t.Errorf("unknown rate must be null: %+v", x)
+	}
+	if x := by[i5.ID]; x.RetiredAt == nil {
+		t.Errorf("retired interface should carry retired_at: %+v", x)
+	}
+	// every nullable field is present as null, not omitted
+	var raw map[string]any
+	_ = json.Unmarshal(rec.Body.Bytes(), &raw)
+	ifs := raw["interfaces"].([]any)[3].(map[string]any)
+	for _, k := range []string{"in_bps", "out_bps", "in_errors_ps", "out_errors_ps", "retired_at"} {
+		if v, ok := ifs[k]; !ok || v != nil {
+			t.Errorf("%s should be present and null: %v %v", k, v, ok)
+		}
+	}
+}
+
+func TestDeviceDetailWithStoreDownStillServes(t *testing.T) {
+	e := newEnv(t, 0)
+	d := e.device(t, "Core", "", "")
+	e.iface(t, d, 1, "e1", "")
+	e.q.err = context.DeadlineExceeded
+	var v api.DeviceView
+	e.getJSON(t, "/api/v1/devices/"+d.Slug, &v)
+	if len(v.Interfaces) != 1 || v.Interfaces[0].InBPS != nil || v.UptimeSeconds != nil {
+		t.Errorf("rates should be null when the store fails: %+v", v)
+	}
+	isProblem(t, e.do("GET", "/api/v1/devices/nope", ""), 404)
+}
+
+func TestPutDeviceEditsAndAudits(t *testing.T) {
+	e := newEnv(t, 0)
+	a := e.device(t, "A", "yyz", "core")
+	b := e.device(t, "B", "yyz", "edge")
+	var v api.DeviceView
+	rec := e.do("PUT", "/api/v1/devices/"+b.Slug,
+		`{"name":"B2","site":"yul","role":"dist","enabled":false,"parents":["`+a.Slug+`"]}`)
+	if rec.Code != 200 {
+		t.Fatalf("PUT: %d %s", rec.Code, rec.Body)
+	}
+	_ = json.Unmarshal(rec.Body.Bytes(), &v)
+	if v.Name != "B2" || v.Site != "yul" || v.Role != "dist" || v.Enabled || len(v.Parents) != 1 || v.Parents[0].Slug != a.Slug {
+		t.Errorf("edited: %+v", v)
+	}
+	if v.Slug != b.Slug {
+		t.Error("slug is immutable")
+	}
+	if e.ref.n.Load() != 1 {
+		t.Error("poller not refreshed")
+	}
+	// a partial body leaves the rest alone; an empty parents list clears them
+	rec = e.do("PUT", "/api/v1/devices/"+b.Slug, `{"parents":[]}`)
+	_ = json.Unmarshal(rec.Body.Bytes(), &v)
+	if rec.Code != 200 || v.Name != "B2" || v.Site != "yul" || len(v.Parents) != 0 {
+		t.Errorf("partial edit: %d %+v", rec.Code, v)
+	}
+	audit, _, _ := e.st.ListAudit(t.Context(), store.Page{})
+	var edits int
+	for _, a := range audit {
+		if a.Action == "device.edit" && a.Object == b.Slug {
+			edits++
+		}
+	}
+	if edits != 2 {
+		t.Errorf("audit has %d device.edit rows, want 2: %+v", edits, audit)
+	}
+	for body, status := range map[string]int{
+		`{"name":""}`: 422, `{"name":"   "}`: 422, `{"parents":["nope"]}`: 422, `{"enabled":"yes"}`: 422, `{"bogus":1}`: 422,
+		`{"snmp":{"version":"v2c"}}`: 422,
+	} {
+		isProblem(t, e.do("PUT", "/api/v1/devices/"+b.Slug, body), status)
+	}
+	isProblem(t, e.do("PUT", "/api/v1/devices/nope", `{}`), 404)
+	if err := e.st.RetireDevice(t.Context(), a.ID); err != nil {
+		t.Fatal(err)
+	}
+	isProblem(t, e.do("PUT", "/api/v1/devices/"+a.Slug, `{"name":"x"}`), 409)
+	isProblem(t, e.do("PUT", "/api/v1/devices/"+b.Slug, `{"parents":["`+a.Slug+`"]}`), 422) // retired parent
+}
+
+func TestPutDeviceRejectsParentCyclesNamingThePath(t *testing.T) {
+	e := newEnv(t, 0)
+	a := e.device(t, "A", "", "")
+	b := e.device(t, "B", "", "")
+	c := e.device(t, "C", "", "")
+	set := func(d *store.Device, parents ...*store.Device) {
+		ids := make([]int64, len(parents))
+		for i, p := range parents {
+			ids[i] = p.ID
+		}
+		if err := e.st.SetParents(t.Context(), d.ID, store.RelNetwork, ids); err != nil {
+			t.Fatal(err)
+		}
+	}
+	set(c, b) // C depends on B
+	set(b, a) // B depends on A
+	rec := e.do("PUT", "/api/v1/devices/"+a.Slug, `{"parents":["`+c.Slug+`"]}`)
+	isProblem(t, rec, 422)
+	want := a.Slug + " -> " + c.Slug + " -> " + b.Slug + " -> " + a.Slug
+	if !strings.Contains(rec.Body.String(), want) {
+		t.Errorf("cycle path missing; want %q in %s", want, rec.Body)
+	}
+	rec = e.do("PUT", "/api/v1/devices/"+a.Slug, `{"parents":["`+a.Slug+`"]}`)
+	isProblem(t, rec, 422)
+	if !strings.Contains(rec.Body.String(), a.Slug+" -> "+a.Slug) {
+		t.Errorf("self parent: %s", rec.Body)
+	}
+	if got, _ := e.st.ListParents(t.Context(), a.ID, ""); len(got) != 0 {
+		t.Errorf("a rejected edit must not change parents: %+v", got)
+	}
+	// a diamond is not a cycle
+	d := e.device(t, "D", "", "")
+	if rec := e.do("PUT", "/api/v1/devices/"+d.Slug, `{"parents":["`+b.Slug+`","`+c.Slug+`"]}`); rec.Code != 200 {
+		t.Errorf("diamond: %d %s", rec.Code, rec.Body)
+	}
+}
+
+func TestPutDeviceReplacesSNMPCredentialWithoutLeakingIt(t *testing.T) {
+	e := newEnv(t, 0)
+	cred := &store.SNMPCredential{Name: "old", Version: "v2c", Community: "old-community-xyz"}
+	if err := e.st.CreateCredential(t.Context(), cred); err != nil {
+		t.Fatal(err)
+	}
+	d := e.device(t, "A", "", "", func(d *store.Device) { d.SNMPCredentialID = &cred.ID })
+	rec := e.do("PUT", "/api/v1/devices/"+d.Slug,
+		`{"snmp":{"version":"v3","v3_user":"nmon","v3_auth_proto":"SHA256","v3_auth_pass":"auth-secret-123","v3_priv_proto":"AES","v3_priv_pass":"priv-secret-456","port":1161}}`)
+	if rec.Code != 200 {
+		t.Fatalf("PUT: %d %s", rec.Code, rec.Body)
+	}
+	assertNoLeaks(t, "PUT response", rec.Body.Bytes(), "auth-secret-123", "priv-secret-456", "old-community-xyz")
+	var v api.DeviceView
+	_ = json.Unmarshal(rec.Body.Bytes(), &v)
+	if v.SNMPVersion != "v3" {
+		t.Errorf("snmp_version %q", v.SNMPVersion)
+	}
+	got, err := e.st.GetDeviceBySlug(t.Context(), d.Slug)
+	if err != nil || got.SNMPPort != 1161 || *got.SNMPCredentialID == cred.ID {
+		t.Fatalf("device after: %+v %v", got, err)
+	}
+	c, err := e.st.GetCredential(t.Context(), *got.SNMPCredentialID)
+	if err != nil || c.V3AuthPass != "auth-secret-123" || c.V3User != "nmon" {
+		t.Errorf("stored credential: %+v %v", c, err)
+	}
+	if _, err := e.st.GetCredential(t.Context(), cred.ID); err == nil {
+		t.Error("the replaced credential should be deleted")
+	}
+	audit, _, _ := e.st.ListAudit(t.Context(), store.Page{})
+	for _, a := range audit {
+		if strings.Contains(a.After+a.Before, "secret") {
+			t.Errorf("audit row leaks a secret: %+v", a)
+		}
+	}
+	vm := e.device(t, "VM", "", "", func(d *store.Device) { d.Kind = store.KindVM })
+	isProblem(t, e.do("PUT", "/api/v1/devices/"+vm.Slug, `{"snmp":{"version":"v2c","community":"c"}}`), 422)
+}

internal/api/device_put.go

@@ -0,0 +1,144 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"strings"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// PutDeviceBody is the body of PUT /api/v1/devices/{slug}. Absent fields stay
+// as they are; an empty parents list removes every parent.
+type PutDeviceBody struct {
+	Name    *string   `json:"name,omitempty" minLength:"1" maxLength:"255"`
+	Site    *string   `json:"site,omitempty"`
+	Role    *string   `json:"role,omitempty"`
+	Enabled *bool     `json:"enabled,omitempty"`
+	Parents []string  `json:"parents,omitempty" doc:"Parent device slugs, replacing the current parents."`
+	SNMP    *SNMPBody `json:"snmp,omitempty" doc:"Replaces the SNMP credential; secrets are write-only."`
+}
+
+func isNotFound(err error) bool { return errors.Is(err, store.ErrNotFound) }
+
+func (s *server) putDevice(ctx context.Context, in *struct {
+	Slug string `path:"slug" doc:"Device slug."`
+	Body PutDeviceBody
+}) (*deviceOut, error) {
+	dev, err := s.Store.GetDeviceBySlug(ctx, in.Slug)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	if dev.PurgedAt != nil {
+		return nil, huma.Error404NotFound("not found")
+	}
+	if dev.RetiredAt != nil {
+		return nil, huma.Error409Conflict("device is retired; reattach it before editing")
+	}
+	plan, err := s.planEdit(ctx, dev, in.Body)
+	if err != nil {
+		return nil, err
+	}
+	if err := s.applyEdit(ctx, dev, plan); err != nil {
+		return nil, s.storeErr(err)
+	}
+	s.Poller.Refresh()
+	return s.view(ctx, dev.Slug)
+}
+
+// editPlan is a validated edit: the device as it will be, the parents to set
+// (nil leaves them) and the credential to create (nil leaves it).
+type editPlan struct {
+	after      store.Device
+	setParents bool
+	parents    []*store.Device
+	cred       *store.SNMPCredential
+	change     map[string]any // what the audit row records; never a secret
+}
+
+// planEdit validates body against the device without writing anything.
+func (s *server) planEdit(ctx context.Context, dev *store.Device, b PutDeviceBody) (*editPlan, error) {
+	p := &editPlan{after: *dev, change: map[string]any{}}
+	set := func(field string, dst *string, v *string) {
+		if v != nil && *v != *dst {
+			*dst = *v
+			p.change[field] = *v
+		}
+	}
+	set("name", &p.after.Name, b.Name)
+	set("site", &p.after.Site, b.Site)
+	set("role", &p.after.Role, b.Role)
+	if b.Name != nil && strings.TrimSpace(*b.Name) == "" {
+		return nil, huma.Error422UnprocessableEntity("name must not be empty")
+	}
+	if b.Enabled != nil && *b.Enabled != dev.Enabled {
+		p.after.Enabled = *b.Enabled
+		p.change["enabled"] = *b.Enabled
+	}
+	if b.Parents != nil {
+		parents, err := s.resolveParents(ctx, dev, b.Parents)
+		if err != nil {
+			return nil, err
+		}
+		p.setParents, p.parents = true, parents
+		p.change["parents"] = b.Parents
+	}
+	if b.SNMP != nil {
+		if dev.Kind != store.KindNetwork {
+			return nil, huma.Error422UnprocessableEntity("snmp applies to network devices only")
+		}
+		cred, err := s.credential(b.SNMP)
+		if err != nil {
+			return nil, err
+		}
+		p.cred = cred
+		p.after.SNMPPort = b.SNMP.Port
+		p.change["snmp"] = map[string]any{"version": b.SNMP.Version, "port": b.SNMP.Port, "replaced": true}
+	}
+	return p, nil
+}
+
+// applyEdit writes the plan and its audit row in one transaction. The old
+// credential is deleted once nothing uses it.
+func (s *server) applyEdit(ctx context.Context, dev *store.Device, p *editPlan) error {
+	return s.Store.WithTx(ctx, func(tx store.Store) error {
+		after := p.after
+		oldCred := dev.SNMPCredentialID
+		if p.cred != nil {
+			if err := tx.CreateCredential(ctx, p.cred); err != nil {
+				return err
+			}
+			after.SNMPCredentialID = &p.cred.ID
+		}
+		if err := tx.UpdateDevice(ctx, &after); err != nil {
+			return err
+		}
+		if p.setParents {
+			if err := setParentLinks(ctx, tx, &after, p.parents); err != nil {
+				return err
+			}
+		}
+		if p.cred != nil && oldCred != nil {
+			if err := tx.DeleteCredential(ctx, *oldCred); err != nil && !errors.Is(err, store.ErrConflict) {
+				return err
+			}
+		}
+		return tx.AppendAudit(ctx, "anonymous", "device.edit", dev.Slug, nil, p.change)
+	})
+}
+
+// setParentLinks replaces the device's parents. VMs and containers depend on
+// their host (hosted_on); everything else on the network path.
+func setParentLinks(ctx context.Context, tx store.Store, d *store.Device, parents []*store.Device) error {
+	rel := store.RelNetwork
+	if d.Kind == store.KindVM || d.Kind == store.KindContainer {
+		rel = store.RelHostedOn
+	}
+	ids := make([]int64, len(parents))
+	for i, p := range parents {
+		ids[i] = p.ID
+	}
+	return tx.SetParents(ctx, d.ID, rel, ids)
+}

internal/api/device_view.go

@@ -0,0 +1,210 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// DeviceRef names a device in another object. Devices are addressed by slug.
+type DeviceRef struct {
+	Slug string `json:"slug"`
+	Name string `json:"name"`
+}
+
+// GroupView is a group an interface of the device belongs to.
+type GroupView struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+	Kind string `json:"kind" enum:"site,customer,role,custom"`
+}
+
+// InterfaceView is an interface as the API shows it, addressed by ULID. The
+// rate fields are five-minute rates from the hot store, null when unknown or
+// when the interface is unreliable (its counters cannot be trusted).
+type InterfaceView struct {
+	ID             string     `json:"id" doc:"Immutable ULID."`
+	IfIndex        int64      `json:"ifindex" doc:"Current ifIndex; may change, the ID does not."`
+	IfName         string     `json:"ifname"`
+	IfDescr        string     `json:"ifdescr"`
+	IfAlias        string     `json:"ifalias"`
+	IfPhysAddr     string     `json:"ifphysaddr"`
+	IfSpeed        int64      `json:"ifspeed" doc:"bits per second"`
+	IfHighSpeed    int64      `json:"ifhighspeed" doc:"megabits per second"`
+	IfType         int64      `json:"iftype"`
+	AdminStatus    int64      `json:"admin_status" doc:"ifAdminStatus: 1 up, 2 down, 3 testing"`
+	OperStatus     int64      `json:"oper_status" doc:"ifOperStatus: 1 up, 2 down, 3 testing, 4 unknown, 5 dormant, 6 notPresent, 7 lowerLayerDown"` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
+	HasHCCounters  bool       `json:"has_hc_counters"`
+	Excluded       bool       `json:"excluded"`
+	FirstSeen      time.Time  `json:"first_seen"`
+	LastDiscovered time.Time  `json:"last_discovered"`
+	Unreliable     bool       `json:"unreliable" doc:"Counters cannot be trusted (32-bit counters on a fast link); no rate is reported."`
+	RetiredAt      *time.Time `json:"retired_at" nullable:"true"`
+	InBPS          *float64   `json:"in_bps" nullable:"true" doc:"Inbound bits per second over 5 minutes."`
+	OutBPS         *float64   `json:"out_bps" nullable:"true"`
+	InErrorsPS     *float64   `json:"in_errors_ps" nullable:"true" doc:"Inbound errors per second over 5 minutes."`
+	OutErrorsPS    *float64   `json:"out_errors_ps" nullable:"true"`
+}
+
+// DeviceView is a device as the API shows it. It carries no numeric ID and no credential.
+type DeviceView struct {
+	Slug          string          `json:"slug" doc:"Immutable public key."`
+	Name          string          `json:"name"`
+	Kind          string          `json:"kind"`
+	Site          string          `json:"site"`
+	Role          string          `json:"role"`
+	Profile       string          `json:"profile"`
+	Enabled       bool            `json:"enabled"`
+	MgmtIP        string          `json:"mgmt_ip"`
+	Addresses     []string        `json:"addresses"`
+	SNMPVersion   string          `json:"snmp_version,omitempty"`
+	SysName       string          `json:"sys_name"`
+	SysObjectID   string          `json:"sys_object_id"`
+	SysDescr      string          `json:"sys_descr"`
+	HWSerial      string          `json:"hw_serial"`
+	UptimeSeconds *float64        `json:"uptime_seconds" nullable:"true" doc:"sysUpTime from the hot store; null when unknown."`
+	Reachability  string          `json:"reachability" enum:"up,down,warning,unknown,maintenance"`
+	PollState     string          `json:"poll_state" enum:"ok,degraded,backoff"`
+	Warnings      []string        `json:"warnings"`
+	LastSeen      *time.Time      `json:"last_seen" nullable:"true"`
+	CreatedAt     time.Time       `json:"created_at"`
+	RetiredAt     *time.Time      `json:"retired_at,omitempty" nullable:"true"`
+	Parents       []DeviceRef     `json:"parents"`
+	Groups        []GroupView     `json:"groups"`
+	Interfaces    []InterfaceView `json:"interfaces"`
+}
+
+type deviceOut struct{ Body DeviceView }
+
+func (s *server) registerDeviceViews(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "get-device", Method: http.MethodGet, Path: "/api/v1/devices/{slug}", Tags: []string{"devices"},
+		Summary:     "Get a device, its interfaces with live rates, parents and groups",
+		Description: "Rates are one batched query to the hot store; they are null when the store is unavailable.",
+		Errors:      []int{http.StatusNotFound},
+	}, s.getDevice)
+	huma.Register(api, huma.Operation{
+		OperationID: "put-device", Method: http.MethodPut, Path: "/api/v1/devices/{slug}", Tags: []string{"devices"},
+		Summary: "Edit a device",
+		Description: "Only the fields present change. parents replaces the parent list (by slug) and is rejected with 422 " +
+			"naming the path when it would make a cycle. snmp replaces the credential.",
+		Errors: []int{http.StatusNotFound, http.StatusConflict, http.StatusUnprocessableEntity},
+	}, s.putDevice)
+}
+
+func (s *server) getDevice(ctx context.Context, in *struct {
+	Slug string `path:"slug" doc:"Device slug."`
+}) (*deviceOut, error) {
+	return s.view(ctx, in.Slug)
+}
+
+// view loads the device and everything shown with it.
+func (s *server) view(ctx context.Context, slug string) (*deviceOut, error) {
+	d, err := s.Store.GetDeviceBySlug(ctx, slug)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	if d.PurgedAt != nil {
+		return nil, huma.Error404NotFound("not found")
+	}
+	v := DeviceView{
+		Slug: d.Slug, Name: d.Name, Kind: d.Kind, Site: d.Site, Role: d.Role, Profile: d.Profile, Enabled: d.Enabled,
+		MgmtIP: d.MgmtIP, SysName: d.SysName, SysObjectID: d.SysObjectID, SysDescr: d.SysDescr, HWSerial: d.HWSerial,
+		PollState: pollStateOf(d), Warnings: nonNil(d.Warnings), LastSeen: utcp(d.LastSeen), CreatedAt: d.CreatedAt.UTC(),
+		RetiredAt: utcp(d.RetiredAt), Addresses: []string{}, Parents: []DeviceRef{}, Groups: []GroupView{},
+		Reachability: s.Reachability.Reachability(ctx, []store.Device{*d})[d.Slug],
+	}
+	if err := s.fillRelations(ctx, d, &v); err != nil {
+		return nil, s.storeErr(err)
+	}
+	if err := s.fillInterfaces(ctx, d, &v); err != nil {
+		return nil, s.storeErr(err)
+	}
+	return &deviceOut{Body: v}, nil
+}
+
+// fillRelations adds addresses, the credential's version (never its secrets), parents and groups.
+func (s *server) fillRelations(ctx context.Context, d *store.Device, v *DeviceView) error {
+	addrs, err := s.Store.ListAddresses(ctx, d.ID)
+	if err != nil {
+		return err
+	}
+	for _, a := range addrs {
+		v.Addresses = append(v.Addresses, a.Addr)
+	}
+	if d.SNMPCredentialID != nil {
+		c, err := s.Store.GetCredential(ctx, *d.SNMPCredentialID)
+		if err != nil {
+			return err
+		}
+		v.SNMPVersion = c.Version
+	}
+	links, err := s.Store.ListParents(ctx, d.ID, "")
+	if err != nil {
+		return err
+	}
+	seen := map[int64]bool{}
+	for _, l := range links {
+		if seen[l.ParentID] {
+			continue
+		}
+		seen[l.ParentID] = true
+		p, err := s.Store.GetDeviceByID(ctx, l.ParentID)
+		if err != nil {
+			return err
+		}
+		v.Parents = append(v.Parents, DeviceRef{Slug: p.Slug, Name: p.Name})
+	}
+	groups, err := s.Store.DeviceGroups(ctx, d.ID)
+	if err != nil {
+		return err
+	}
+	for _, g := range groups {
+		v.Groups = append(v.Groups, GroupView{ID: itoa(int(g.ID)), Name: g.Name, Kind: g.Kind})
+	}
+	return nil
+}
+
+// fillInterfaces adds the device's interfaces, retired ones included, with live rates.
+func (s *server) fillInterfaces(ctx context.Context, d *store.Device, v *DeviceView) error {
+	ifs, err := s.Store.ListInterfaces(ctx, d.ID, true)
+	if err != nil {
+		return err
+	}
+	var rates map[string]*ifRates
+	v.Interfaces = make([]InterfaceView, 0, len(ifs))
+	if len(ifs) > 0 || d.Kind == store.KindNetwork {
+		rates, v.UptimeSeconds = s.liveRates(ctx, `device=`+quote(d.Slug), true)
+	}
+	for i := range ifs {
+		v.Interfaces = append(v.Interfaces, interfaceView(&ifs[i], rates[ifs[i].ID]))
+	}
+	return nil
+}
+
+func interfaceView(i *store.Interface, r *ifRates) InterfaceView {
+	in, out := r.bps(i)
+	inE, outE := r.errs(i)
+	return InterfaceView{
+		ID: i.ID, IfIndex: i.IfIndex, IfName: i.IfName, IfDescr: i.IfDescr, IfAlias: i.IfAlias,
+		IfPhysAddr: i.IfPhysAddr, IfSpeed: i.IfSpeed, IfHighSpeed: i.IfHighSpeed, IfType: i.IfType,
+		AdminStatus: i.AdminStatus, OperStatus: i.OperStatus, HasHCCounters: i.HasHCCounters, Excluded: i.Excluded,
+		FirstSeen: i.FirstSeen.UTC(), LastDiscovered: i.LastDiscovered.UTC(), Unreliable: i.Unreliable,
+		RetiredAt: utcp(i.RetiredAt), InBPS: in, OutBPS: out, InErrorsPS: inE, OutErrorsPS: outE,
+	}
+}
+
+// unreliableReason says why an interface's counters cannot be trusted.
+func unreliableReason(i *store.Interface) string {
+	switch {
+	case !i.HasHCCounters:
+		return "counter_32bit_fast_link"
+	case i.CounterDiscontinuity > 0:
+		return "counter_discontinuity"
+	}
+	return "counter_unreliable"
+}

internal/api/devices.go

@@ -53,54 +53,10 @@ type reattachCandidate struct {
 
 // reattachProblem is the 409 problem with the retired devices the client may restore.
 type reattachProblem struct {
-	huma.ErrorModel
+	Problem
 	Candidates []reattachCandidate `json:"reattach_candidates"`
 }
 
-// InterfaceView is an interface as the API shows it, addressed by ULID.
-type InterfaceView struct {
-	ID             string    `json:"id" doc:"Immutable ULID."`
-	IfIndex        int64     `json:"ifindex" doc:"Current ifIndex; may change, the ID does not."`
-	IfName         string    `json:"ifname"`
-	IfDescr        string    `json:"ifdescr"`
-	IfAlias        string    `json:"ifalias"`
-	IfPhysAddr     string    `json:"ifphysaddr"`
-	IfSpeed        int64     `json:"ifspeed" doc:"bits per second"`
-	IfHighSpeed    int64     `json:"ifhighspeed" doc:"megabits per second"`
-	IfType         int64     `json:"iftype"`
-	AdminStatus    int64     `json:"admin_status"`
-	OperStatus     int64     `json:"oper_status"`
-	HasHCCounters  bool      `json:"has_hc_counters"`
-	Excluded       bool      `json:"excluded"`
-	FirstSeen      time.Time `json:"first_seen"`
-	LastDiscovered time.Time `json:"last_discovered"`
-}
-
-// DeviceView is a device as the API shows it. It carries no numeric ID and no credential.
-type DeviceView struct {
-	Slug        string          `json:"slug" doc:"Immutable public key."`
-	Name        string          `json:"name"`
-	Kind        string          `json:"kind"`
-	Site        string          `json:"site"`
-	Role        string          `json:"role"`
-	Profile     string          `json:"profile"`
-	Enabled     bool            `json:"enabled"`
-	MgmtIP      string          `json:"mgmt_ip"`
-	Addresses   []string        `json:"addresses"`
-	SNMPVersion string          `json:"snmp_version,omitempty"`
-	SysName     string          `json:"sys_name"`
-	SysObjectID string          `json:"sys_object_id"`
-	SysDescr    string          `json:"sys_descr"`
-	PollState   string          `json:"poll_state"`
-	Warnings    []string        `json:"warnings"`
-	LastSeen    *time.Time      `json:"last_seen,omitempty"`
-	CreatedAt   time.Time       `json:"created_at"`
-	RetiredAt   *time.Time      `json:"retired_at,omitempty"`
-	Interfaces  []InterfaceView `json:"interfaces"`
-}
-
-type deviceOut struct{ Body DeviceView }
-
 func (s *server) registerDevices(api huma.API) {
 	huma.Register(api, huma.Operation{
 		OperationID: "add-device", Method: "POST", Path: "/api/v1/devices", Tags: []string{"devices"},
@@ -110,11 +66,6 @@ func (s *server) registerDevices(api huma.API) {
 		DefaultStatus: http.StatusCreated,
 		Errors:        []int{http.StatusConflict, http.StatusUnprocessableEntity, http.StatusBadGateway},
 	}, s.addDevice)
-	huma.Register(api, huma.Operation{
-		OperationID: "get-device", Method: "GET", Path: "/api/v1/devices/{slug}", Tags: []string{"devices"},
-		Summary: "Get a device and its interfaces",
-		Errors:  []int{404},
-	}, s.getDevice)
 }
 
 // addRequest is an AddDeviceBody after validation: defaults applied and
@@ -371,55 +322,6 @@ func (s *server) collect(ctx context.Context, addr netip.Addr, port int, cred *s
 	return res, nil
 }
 
-func (s *server) getDevice(ctx context.Context, in *struct {
-	Slug string `path:"slug" doc:"Device slug."`
-}) (*deviceOut, error) {
-	return s.view(ctx, in.Slug)
-}
-
-func (s *server) view(ctx context.Context, slug string) (*deviceOut, error) {
-	d, err := s.Store.GetDeviceBySlug(ctx, slug)
-	if err != nil {
-		return nil, s.storeErr(err)
-	}
-	if d.PurgedAt != nil {
-		return nil, huma.Error404NotFound("not found")
-	}
-	v := DeviceView{
-		Slug: d.Slug, Name: d.Name, Kind: d.Kind, Site: d.Site, Role: d.Role, Profile: d.Profile, Enabled: d.Enabled,
-		MgmtIP: d.MgmtIP, SysName: d.SysName, SysObjectID: d.SysObjectID, SysDescr: d.SysDescr,
-		PollState: d.PollState, Warnings: nonNil(d.Warnings), LastSeen: utcp(d.LastSeen), CreatedAt: d.CreatedAt.UTC(),
-		RetiredAt: utcp(d.RetiredAt), Addresses: []string{}, Interfaces: []InterfaceView{},
-	}
-	addrs, err := s.Store.ListAddresses(ctx, d.ID)
-	if err != nil {
-		return nil, s.storeErr(err)
-	}
-	for _, a := range addrs {
-		v.Addresses = append(v.Addresses, a.Addr)
-	}
-	if d.SNMPCredentialID != nil {
-		c, err := s.Store.GetCredential(ctx, *d.SNMPCredentialID)
-		if err != nil {
-			return nil, s.storeErr(err)
-		}
-		v.SNMPVersion = c.Version // the secrets stay behind
-	}
-	ifs, err := s.Store.ListInterfaces(ctx, d.ID, false)
-	if err != nil {
-		return nil, s.storeErr(err)
-	}
-	for _, i := range ifs {
-		v.Interfaces = append(v.Interfaces, InterfaceView{
-			ID: i.ID, IfIndex: i.IfIndex, IfName: i.IfName, IfDescr: i.IfDescr, IfAlias: i.IfAlias,
-			IfPhysAddr: i.IfPhysAddr, IfSpeed: i.IfSpeed, IfHighSpeed: i.IfHighSpeed, IfType: i.IfType,
-			AdminStatus: i.AdminStatus, OperStatus: i.OperStatus, HasHCCounters: i.HasHCCounters, Excluded: i.Excluded,
-			FirstSeen: i.FirstSeen.UTC(), LastDiscovered: i.LastDiscovered.UTC(),
-		})
-	}
-	return &deviceOut{Body: v}, nil
-}
-
 func nonNil(s []string) []string {
 	if s == nil {
 		return []string{}

internal/api/devices_list.go

@@ -0,0 +1,136 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// DeviceSummary is one row of the device list.
+type DeviceSummary struct {
+	Slug            string     `json:"slug"`
+	Name            string     `json:"name"`
+	Kind            string     `json:"kind" enum:"network,vm,container,check_only"`
+	Site            string     `json:"site"`
+	Role            string     `json:"role"`
+	MgmtIP          string     `json:"mgmt_ip"`
+	Profile         string     `json:"profile"`
+	Reachability    string     `json:"reachability" enum:"up,down,warning,unknown,maintenance"`
+	PollState       string     `json:"poll_state" enum:"ok,degraded,backoff"`
+	LastSeen        *time.Time `json:"last_seen" nullable:"true"`
+	InterfacesTotal int64      `json:"interfaces_total" doc:"Active interfaces, excluded ones left out."`
+	InterfacesDown  int64      `json:"interfaces_down" doc:"Active interfaces that are admin up and not oper up."`
+	Warnings        []string   `json:"warnings"`
+}
+
+// DeviceList is a page of DeviceSummary.
+type DeviceList struct {
+	Items   []DeviceSummary `json:"items"`
+	Page    int64           `json:"page"`
+	PerPage int64           `json:"per_page"`
+	Total   int64           `json:"total"`
+}
+
+// DeviceFacets are the distinct values the list's filter dropdowns offer.
+type DeviceFacets struct {
+	Sites []string `json:"sites"`
+	Roles []string `json:"roles"`
+	Kinds []string `json:"kinds"`
+}
+
+func (s *server) registerDeviceList(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "list-devices", Method: http.MethodGet, Path: "/api/v1/devices", Tags: []string{"devices"},
+		Summary: "List devices",
+		Description: "Paged, with reachability and interface tallies. state filters on the derived reachability, " +
+			"so it is applied after the other filters.",
+	}, s.listDevices)
+	huma.Register(api, huma.Operation{
+		OperationID: "get-device-facets", Method: http.MethodGet, Path: "/api/v1/devices/facets", Tags: []string{"devices"},
+		Summary: "Distinct site, role and kind values of unretired devices",
+	}, s.deviceFacets)
+}
+
+func (s *server) listDevices(ctx context.Context, in *struct {
+	Site  string `query:"site"`
+	Role  string `query:"role"`
+	Kind  string `query:"kind" enum:"network,vm,container,check_only"`
+	Q     string `query:"q" doc:"Substring of name, slug, management address or sysName."`
+	State string `query:"state" enum:"up,down,warning,unknown,maintenance" doc:"Reachability."`
+	PageParams
+}) (*struct{ Body DeviceList }, error) {
+	f := store.DeviceFilter{Site: in.Site, Role: in.Role, Kind: in.Kind, Q: in.Q}
+	var (
+		devs  []store.Device
+		total int64
+		reach map[string]string
+		err   error
+	)
+	if in.State == "" {
+		devs, total, err = s.Store.ListDevices(ctx, f, store.Page{Offset: in.offset(), Limit: in.limit()})
+		if err == nil {
+			reach = s.Reachability.Reachability(ctx, devs)
+		}
+	} else {
+		devs, total, reach, err = s.devicesInState(ctx, f, in.State, in.PageParams)
+	}
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	counts, err := s.Store.InterfaceCounts(ctx)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	out := DeviceList{Items: make([]DeviceSummary, 0, len(devs)), Page: in.pageNo(), PerPage: in.limit(), Total: total}
+	for i := range devs {
+		d := &devs[i]
+		c := counts[d.ID]
+		out.Items = append(out.Items, DeviceSummary{
+			Slug: d.Slug, Name: d.Name, Kind: d.Kind, Site: d.Site, Role: d.Role, MgmtIP: d.MgmtIP, Profile: d.Profile,
+			Reachability: reach[d.Slug], PollState: pollStateOf(d), LastSeen: utcp(d.LastSeen),
+			InterfacesTotal: c.Total, InterfacesDown: c.Down, Warnings: nonNil(d.Warnings),
+		})
+	}
+	return &struct{ Body DeviceList }{out}, nil
+}
+
+// devicesInState pages over the devices whose derived reachability is state.
+// Reachability is not a column, so every device matching the other filters is
+// classified first and the page is cut from the survivors.
+func (s *server) devicesInState(
+	ctx context.Context, f store.DeviceFilter, state string, p PageParams,
+) (page []store.Device, total int64, reach map[string]string, err error) {
+	var all []store.Device
+	for off := int64(0); ; off += store.MaxLimit {
+		rows, n, err := s.Store.ListDevices(ctx, f, store.Page{Offset: off, Limit: store.MaxLimit})
+		if err != nil {
+			return nil, 0, nil, err
+		}
+		all = append(all, rows...)
+		if off+store.MaxLimit >= n {
+			break
+		}
+	}
+	reach = s.Reachability.Reachability(ctx, all)
+	var match []store.Device
+	for _, d := range all {
+		if reach[d.Slug] == state {
+			match = append(match, d)
+		}
+	}
+	lo := min(p.offset(), int64(len(match)))
+	hi := min(lo+p.limit(), int64(len(match)))
+	return match[lo:hi], int64(len(match)), reach, nil
+}
+
+func (s *server) deviceFacets(ctx context.Context, _ *struct{}) (*struct{ Body DeviceFacets }, error) {
+	sites, roles, kinds, err := s.Store.DeviceFacets(ctx)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	return &struct{ Body DeviceFacets }{DeviceFacets{Sites: nonNil(sites), Roles: nonNil(roles), Kinds: nonNil(kinds)}}, nil
+}

internal/api/docs.go

@@ -0,0 +1,31 @@
+package api
+
+import (
+	"embed"
+	"io/fs"
+	"net/http"
+)
+
+// docsFS holds the Swagger UI page and its vendored swagger-ui-dist 5.17.14
+// assets, so /docs works on a network with no internet access (huma's own
+// renderers load their JavaScript from a CDN).
+//
+//go:embed docsui
+var docsFS embed.FS
+
+// registerDocs serves Swagger UI at /docs and its assets under /docs/assets/.
+func registerDocs(mux *http.ServeMux) {
+	sub, err := fs.Sub(docsFS, "docsui")
+	if err != nil {
+		panic("api: embedded docs missing: " + err.Error()) // the embed directive guarantees the directory
+	}
+	page, err := fs.ReadFile(sub, "index.html")
+	if err != nil {
+		panic("api: embedded docs page missing: " + err.Error())
+	}
+	mux.HandleFunc("GET /docs", func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write(page)
+	})
+	mux.Handle("GET /docs/assets/", http.StripPrefix("/docs/assets/", http.FileServerFS(sub)))
+}

internal/api/docsui/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

internal/api/docsui/NOTICE

@@ -0,0 +1,2 @@
+swagger-ui
+Copyright 2020-2021 SmartBear Software Inc.

internal/api/docsui/index.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>nmon API</title>
+<link rel="stylesheet" href="/docs/assets/swagger-ui.css">
+</head>
+<body>
+<div id="swagger-ui"></div>
+<script src="/docs/assets/swagger-ui-bundle.js"></script>
+<script src="/docs/assets/init.js"></script>
+</body>
+</html>

internal/api/docsui/init.js

@@ -0,0 +1 @@
+window.ui = SwaggerUIBundle({ url: '/openapi.json', dom_id: '#swagger-ui', deepLinking: true });

internal/api/graph.go

@@ -0,0 +1,142 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+func (s *server) registerGraph(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "get-graph", Method: http.MethodGet, Path: "/api/v1/graph", Tags: []string{"graph"},
+		Summary: "Bits per second in and out of one interface",
+		Description: "From the hot store when the range lies within hot retention; older data comes from the long store's " +
+			"hourly rollups, and a range that needs both is stitched at stitched_at. Stretches with no data in either store " +
+			"are listed in partial. An unreliable interface returns no series. max=true adds the per-step maximum; " +
+			"p95=true adds the 95th percentile of 5-minute averages, computed live.",
+		Errors: []int{http.StatusNotFound, http.StatusUnprocessableEntity, http.StatusTooManyRequests, http.StatusServiceUnavailable},
+	}, s.graph)
+}
+
+type graphIn struct {
+	Iface string    `query:"iface" required:"true" minLength:"1" doc:"Interface ULID."`
+	From  time.Time `query:"from" doc:"Start, RFC 3339. Default one hour before to."`
+	To    time.Time `query:"to" doc:"End, RFC 3339. Default now."`
+	Step  int64     `query:"step" minimum:"1" doc:"Step in seconds. Default max(60, range/1000) rounded up to whole minutes."`
+	Dir   string    `query:"dir" enum:"in,out" doc:"Only this direction; the other series is empty."`
+	Max   bool      `query:"max" doc:"Also return in_max and out_max, the highest one-minute rate within each step."`
+	P95   bool      `query:"p95" doc:"Also return the live 95th percentile of 5-minute averages."`
+}
+
+func (s *server) graph(ctx context.Context, in *graphIn) (*struct{ Body GraphResponse }, error) {
+	i, err := s.Store.GetInterface(ctx, in.Iface)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	dev, err := s.Store.GetDeviceByID(ctx, i.DeviceID)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	from, to, err := s.graphRange(in)
+	if err != nil {
+		return nil, err
+	}
+	resp := GraphResponse{
+		Iface: GraphIface{
+			ID: i.ID, IfName: i.IfName, IfAlias: i.IfAlias, SpeedBPS: speedBPS(i), Device: DeviceRef{Slug: dev.Slug, Name: dev.Name},
+		},
+		Partial: []TimeSpan{}, Series: GraphSeries{In: []Point{}, Out: []Point{}},
+	}
+	hotStart := s.Clock.Now().Add(-s.HotRetention)
+	plan, err := planGraph(from, to, hotStart, time.Duration(in.Step)*time.Second)
+	if err != nil {
+		return nil, huma.Error422UnprocessableEntity(err.Error())
+	}
+	resp.StepSeconds, resp.Tier = int64(plan.newest().step/time.Second), plan.newest().tier
+	if i.Unreliable {
+		resp.Unreliable, resp.UnreliableReason = true, unreliableReason(i)
+		return &struct{ Body GraphResponse }{resp}, nil
+	}
+	if err := s.fillGraph(ctx, &resp, i, plan, in, hotStart); err != nil {
+		return nil, err
+	}
+	return &struct{ Body GraphResponse }{resp}, nil
+}
+
+// graphRange applies the defaults and validates the requested range.
+func (s *server) graphRange(in *graphIn) (from, to time.Time, err error) {
+	to = in.To
+	if to.IsZero() {
+		to = s.Clock.Now()
+	}
+	from = in.From
+	if from.IsZero() {
+		from = to.Add(-time.Hour)
+	}
+	if !from.Before(to) {
+		return from, to, huma.Error422UnprocessableEntity("from must be before to")
+	}
+	return from.UTC(), to.UTC(), nil
+}
+
+// fillGraph queries the stores the plan names and lays the samples out on the grid.
+func (s *server) fillGraph(
+	ctx context.Context, resp *GraphResponse, i *store.Interface, p *graphPlan, in *graphIn, hotStart time.Time,
+) error {
+	release, err := s.admit(ctx, false)
+	if err != nil {
+		return err
+	}
+	defer release()
+	qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
+	defer cancel()
+	q := &graphQuery{iface: i, dirs: []string{dirIn, dirOut}, withMax: in.Max}
+	if in.Dir != "" {
+		q.dirs = []string{in.Dir}
+	}
+	data := newGraphData()
+	for _, w := range p.tiers {
+		if err := s.fetchTier(qctx, q, w, data); err != nil {
+			return err
+		}
+	}
+	assemble(resp, p, q, data)
+	if in.P95 {
+		p95, err := s.livep95(qctx, q, p, hotStart)
+		if err != nil {
+			return err
+		}
+		resp.P95 = p95
+	}
+	return nil
+}
+
+// assemble turns the samples into series, tiers, stitched_at and partial spans.
+func assemble(resp *GraphResponse, p *graphPlan, q *graphQuery, d *graphData) {
+	grid := gridOf(p)
+	for _, dir := range q.dirs {
+		resp.Series.set(dir, false, pointsOf(grid, d.avg[dir]))
+		if q.withMax {
+			resp.Series.set(dir, true, pointsOf(grid, d.max[dir]))
+		}
+	}
+	has := func(ts int64) bool {
+		for _, dir := range q.dirs {
+			if _, ok := d.avg[dir][ts]; ok {
+				return true
+			}
+		}
+		return false
+	}
+	resp.Partial = append(resp.Partial, findGaps(grid, has, p.from.Unix(), p.to.Unix())...)
+	if p.stitchedAt != nil {
+		resp.StitchedAt = p.stitchedAt
+		for _, w := range p.tiers {
+			resp.Tiers = append(resp.Tiers, GraphTier{From: w.from.UTC(), To: w.bound.UTC(), Tier: w.tier, StepSeconds: int64(w.step / time.Second)})
+		}
+	}
+}

internal/api/graph_fetch.go

@@ -0,0 +1,163 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/v10networks/nmon/internal/store"
+	"github.com/v10networks/nmon/internal/vmsup"
+)
+
+const (
+	dirIn  = "in"
+	dirOut = "out"
+	// minRateWindow keeps the rate window above two poll intervals so a step of
+	// one minute still sees two samples.
+	minRateWindow = 2 * time.Minute
+	// maxInnerStep is the resolution of the max-over-step subquery.
+	maxInnerStep = time.Minute
+)
+
+// graphData holds the samples found, by direction and Unix second.
+type graphData struct {
+	avg map[string]map[int64]float64
+	max map[string]map[int64]float64
+}
+
+func newGraphData() *graphData {
+	return &graphData{avg: map[string]map[int64]float64{}, max: map[string]map[int64]float64{}}
+}
+
+func (g *graphData) put(isMax bool, dir string, ts int64, v float64) {
+	m := g.avg
+	if isMax {
+		m = g.max
+	}
+	if m[dir] == nil {
+		m[dir] = map[int64]float64{}
+	}
+	if old, ok := m[dir][ts]; !ok || v > old {
+		m[dir][ts] = v // stores agree on duplicates; the highest wins
+	}
+}
+
+// graphQuery is what one fetch needs to know.
+type graphQuery struct {
+	iface   *store.Interface
+	dirs    []string
+	withMax bool
+}
+
+// octets is the counter that carries direction dir for the interface.
+func (q *graphQuery) octets(dir string) string {
+	switch {
+	case q.iface.HasHCCounters && dir == dirIn:
+		return "ifHCInOctets"
+	case q.iface.HasHCCounters:
+		return "ifHCOutOctets"
+	case dir == dirIn:
+		return "ifInOctets"
+	}
+	return "ifOutOctets"
+}
+
+func secs(d time.Duration) string { return strconv.FormatInt(int64(d/time.Second), 10) + "s" }
+
+// rawExpr is the average bits per second of dir over the rate window.
+func (q *graphQuery) rawExpr(dir string, step time.Duration) string {
+	sel := q.octets(dir) + `{iface=` + quote(q.iface.ID) + `}`
+	return `rate(` + sel + `[` + secs(max(step, minRateWindow)) + `])*8`
+}
+
+// rawMaxExpr is the highest one-minute rate seen within each step.
+func (q *graphQuery) rawMaxExpr(dir string, step time.Duration) string {
+	sel := q.octets(dir) + `{iface=` + quote(q.iface.ID) + `}`
+	return `max_over_time((rate(` + sel + `[` + secs(maxInnerStep) + `])*8)[` + secs(step) + `:` + secs(maxInnerStep) + `])`
+}
+
+// longExpr reads an hourly rollup metric. Versions of a rewritten window share
+// the same value, so max by (dir) collapses them; a coarser step averages (or
+// takes the max of) the hours it covers.
+func (q *graphQuery) longExpr(metric, over string, dir string, step time.Duration) string {
+	inner := `max by (dir) (` + metric + `{iface=` + quote(q.iface.ID) + `,dir=` + quote(dir) + `})`
+	if step <= longStepMin {
+		return inner
+	}
+	return over + `((` + inner + `)[` + secs(step) + `:` + secs(longStepMin) + `])`
+}
+
+// fetchTier fills d from the store that owns the tier. Hot errors fail the
+// graph; a long store that is down only leaves its stretch empty, which the
+// response reports as partial.
+func (s *server) fetchTier(ctx context.Context, q *graphQuery, w tierWindow, d *graphData) error {
+	storeName := "hot"
+	if w.tier == tier1h {
+		storeName = "long"
+	}
+	for _, dir := range q.dirs {
+		for _, wantMax := range []bool{false, true} {
+			if wantMax && !q.withMax {
+				continue
+			}
+			err := s.fetchOne(ctx, storeName, q.exprFor(w, dir, wantMax), w, func(ts int64, v float64) { d.put(wantMax, dir, ts, v) })
+			if err == nil {
+				continue
+			}
+			if storeName == "long" && !errors.Is(err, context.Canceled) {
+				s.Logger.Warn("long store query failed; its stretch is reported partial", "err", err)
+				continue
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+func (q *graphQuery) exprFor(w tierWindow, dir string, wantMax bool) string {
+	switch {
+	case w.tier == tierRaw && wantMax:
+		return q.rawMaxExpr(dir, w.step)
+	case w.tier == tierRaw:
+		return q.rawExpr(dir, w.step)
+	case wantMax:
+		return q.longExpr("nmon_if_rate_bps_max_1h", "max_over_time", dir, w.step)
+	}
+	return q.longExpr("nmon_if_rate_bps_avg_1h", "avg_over_time", dir, w.step)
+}
+
+// fetchOne runs one range query over the tier's window and hands every finite sample to add.
+func (s *server) fetchOne(ctx context.Context, storeName, expr string, w tierWindow, add func(ts int64, v float64)) error {
+	body, err := s.Querier.QueryRange(ctx, storeName, expr, w.from, w.last, w.step)
+	if err != nil {
+		return s.vmErr(ctx, err)
+	}
+	rows, err := parseVM(body)
+	if err != nil {
+		return fmt.Errorf("graph: %w", err)
+	}
+	for _, r := range rows {
+		for _, p := range r.Values {
+			if p.OK {
+				add(p.TS, p.V)
+			}
+		}
+	}
+	return nil
+}
+
+// vmErr maps a metric-store failure to a problem. It is shared by /query and /graph.
+func (s *server) vmErr(ctx context.Context, err error) error {
+	switch {
+	case errors.Is(err, vmsup.ErrStoreDown):
+		return errStoreDown()
+	case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded):
+		return errQueryTimeout()
+	case isBadQuery(err):
+		return badQuery(err)
+	}
+	s.Logger.Warn("query failed", "err", err)
+	return errStoreFailed()
+}

internal/api/graph_internal_test.go

@@ -0,0 +1,160 @@
+package api
+
+import (
+	"math"
+	"testing"
+	"time"
+)
+
+func TestDefaultStepIsMaxOf60sAndRangeOver1000RoundedToMinutes(t *testing.T) {
+	for rng, want := range map[time.Duration]time.Duration{
+		time.Hour: time.Minute, 16 * time.Hour: time.Minute, 17 * time.Hour: 2 * time.Minute,
+		24 * time.Hour: 2 * time.Minute, 30 * 24 * time.Hour: 44 * time.Minute, 365 * 24 * time.Hour: 526 * time.Minute,
+	} {
+		if got := defaultStep(rng); got != want {
+			t.Errorf("defaultStep(%s) = %s, want %s", rng, got, want)
+		}
+	}
+}
+
+func TestPlanGraphTiers(t *testing.T) {
+	now := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
+	hotStart := now.Add(-30 * 24 * time.Hour)
+	cases := []struct {
+		name       string
+		from, to   time.Time
+		tiers      []string
+		stitched   bool
+		wantErr    bool
+		step       time.Duration
+		newestStep time.Duration
+	}{
+		{"1h", now.Add(-time.Hour), now, []string{"raw"}, false, false, 0, time.Minute},
+		{"exactly the hot window", hotStart, now, []string{"raw"}, false, false, 0, 44 * time.Minute},
+		{"a hair beyond hot is still raw", hotStart.Add(-20 * time.Minute), now, []string{"raw"}, false, false, 0, 44 * time.Minute},
+		{"straddles", now.Add(-40 * 24 * time.Hour), now, []string{"1h", "raw"}, true, false, 0, 58 * time.Minute},
+		{"ends inside hot start", now.Add(-60 * 24 * time.Hour), hotStart, []string{"1h"}, false, false, 0, time.Hour},
+		{"old", now.Add(-90 * 24 * time.Hour), now.Add(-60 * 24 * time.Hour), []string{"1h"}, false, false, 0, time.Hour},
+		{"too many points", now.Add(-30 * 24 * time.Hour), now, nil, false, true, time.Second, 0},
+		{"explicit step", now.Add(-time.Hour), now, []string{"raw"}, false, false, 5 * time.Minute, 5 * time.Minute},
+	}
+	for _, tc := range cases {
+		p, err := planGraph(tc.from, tc.to, hotStart, tc.step)
+		if (err != nil) != tc.wantErr {
+			t.Errorf("%s: err %v", tc.name, err)
+			continue
+		}
+		if tc.wantErr {
+			continue
+		}
+		var got []string
+		for _, w := range p.tiers {
+			got = append(got, w.tier)
+		}
+		if len(got) != len(tc.tiers) || (p.stitchedAt != nil) != tc.stitched || p.newest().step != tc.newestStep {
+			t.Errorf("%s: tiers %v stitched %v newest step %s, want %v %v %s", tc.name, got, p.stitchedAt, p.newest().step, tc.tiers, tc.stitched, tc.newestStep)
+			continue
+		}
+		for i := range got {
+			if got[i] != tc.tiers[i] {
+				t.Errorf("%s: tiers %v, want %v", tc.name, got, tc.tiers)
+			}
+		}
+		if tc.stitched {
+			join := *p.stitchedAt
+			if join.Before(hotStart) || join.Sub(hotStart) >= p.newest().step || join.Unix()%int64(p.newest().step/time.Second) != 0 {
+				t.Errorf("%s: join %v", tc.name, join)
+			}
+			if long := p.tiers[0]; long.last.Unix() >= join.Unix() || long.bound != join {
+				t.Errorf("%s: long tier overlaps raw: %+v", tc.name, long)
+			}
+		}
+	}
+}
+
+func TestFindGapsThresholdIsTwoSteps(t *testing.T) {
+	grid := make([]slot, 11)
+	for i := range grid {
+		grid[i] = slot{ts: int64(i * 60), step: 60}
+	}
+	has := func(missing ...int) func(int64) bool {
+		return func(ts int64) bool {
+			for _, m := range missing {
+				if ts == int64(m*60) {
+					return false
+				}
+			}
+			return true
+		}
+	}
+	for _, tc := range []struct {
+		name    string
+		missing []int
+		want    [][2]int64
+	}{
+		{"complete", nil, nil},
+		{"one missing sample", []int{4}, nil},
+		{"two missing samples", []int{4, 5}, [][2]int64{{180, 360}}},
+		{"leading", []int{0, 1, 2}, [][2]int64{{0, 180}}},
+		{"leading short", []int{0, 1}, nil},
+		{"trailing", []int{8, 9, 10}, [][2]int64{{420, 600}}},
+		{"two separate holes", []int{2, 3, 7, 8}, [][2]int64{{60, 240}, {360, 540}}},
+	} {
+		got := findGaps(grid, has(tc.missing...), 0, 600)
+		if len(got) != len(tc.want) {
+			t.Errorf("%s: %v, want %v", tc.name, got, tc.want)
+			continue
+		}
+		for i, g := range got {
+			if g.From.Unix() != tc.want[i][0] || g.To.Unix() != tc.want[i][1] {
+				t.Errorf("%s: gap %d is %v..%v, want %v", tc.name, i, g.From.Unix(), g.To.Unix(), tc.want[i])
+			}
+		}
+	}
+	none := func(int64) bool { return false }
+	if g := findGaps(grid, none, 0, 600); len(g) != 1 || g[0].From.Unix() != 0 || g[0].To.Unix() != 600 {
+		t.Errorf("no data: %v", g)
+	}
+	if g := findGaps(nil, none, 0, 0); g != nil {
+		t.Errorf("empty grid: %v", g)
+	}
+}
+
+func TestNearestRank(t *testing.T) {
+	seq := func(n int) []float64 {
+		out := make([]float64, n)
+		for i := range out {
+			out[i] = float64(n - i) // descending, so sorting matters
+		}
+		return out
+	}
+	for n, want := range map[int]float64{1: 1, 2: 2, 19: 19, 20: 19, 21: 20, 100: 95, 101: 96, 1000: 950} {
+		if got := nearestRank(seq(n), 0.95); got != want {
+			t.Errorf("p95 of 1..%d = %v, want %v", n, got, want)
+		}
+	}
+	if !math.IsNaN(nearestRank(nil, 0.95)) {
+		t.Error("no values should be NaN")
+	}
+	in := []float64{3, 1, 2}
+	nearestRank(in, 0.5)
+	if in[0] != 3 {
+		t.Error("input must not be reordered")
+	}
+}
+
+func TestParentCycleHelpersAndEvidence(t *testing.T) {
+	got := evidenceLines(`{"b":2,"a":"x","c":["y"]}`)
+	if len(got) != 3 || got[0] != "a: x" || got[1] != "b: 2" || got[2] != `c: ["y"]` {
+		t.Errorf("evidence: %q", got)
+	}
+	if got := evidenceLines(`not json`); len(got) != 1 || got[0] != "not json" {
+		t.Errorf("evidence fallback: %q", got)
+	}
+	if got := evidenceLines(``); got == nil || len(got) != 0 {
+		t.Errorf("empty evidence must be [] not nil: %#v", got)
+	}
+	if quote(`a"b\c`) != `"a\"b\\c"` {
+		t.Errorf("quote: %s", quote(`a"b\c`))
+	}
+}

internal/api/graph_p95.go

@@ -0,0 +1,64 @@
+package api
+
+import (
+	"context"
+	"time"
+)
+
+const (
+	p95Quantile = 0.95
+	p95Slot     = 5 * time.Minute
+	// maxP95Span bounds the live computation to what one range query can return (about 25,000 slots).
+	maxP95Span = 85 * 24 * time.Hour
+)
+
+// livep95 computes the 95th percentile of 5-minute average rates over the
+// raw (hot) part of the drawn range, by nearest rank. M5 adds the materialised
+// monthly value; until then the source is "live" and coverage says how much of
+// the range's 5-minute slots had data, so a short history never passes for a full month.
+func (s *server) livep95(ctx context.Context, q *graphQuery, p *graphPlan, hotStart time.Time) (*GraphP95, error) {
+	raw := p.newest()
+	if raw.tier != tierRaw {
+		return &GraphP95{Source: "live"}, nil
+	}
+	start := ceilTo(latest(raw.from, hotStart, raw.bound.Add(-maxP95Span)), p95Slot)
+	end := floorTo(raw.bound, p95Slot)
+	expected := max(int64(p.to.Sub(p.from)/p95Slot), 1)
+	out := &GraphP95{Source: "live"}
+	if end.Before(start) {
+		return out, nil
+	}
+	w := tierWindow{tier: tierRaw, from: start, last: end, bound: end, step: p95Slot}
+	coverage := 1.0
+	for _, dir := range q.dirs {
+		var vals []float64
+		expr := `rate(` + q.octets(dir) + `{iface=` + quote(q.iface.ID) + `}[` + secs(p95Slot) + `])*8`
+		err := s.fetchOne(ctx, "hot", expr, w, func(_ int64, v float64) { vals = append(vals, v) })
+		if err != nil {
+			return nil, err
+		}
+		coverage = min(coverage, float64(len(vals))/float64(expected))
+		if len(vals) == 0 {
+			continue
+		}
+		v := nearestRank(vals, p95Quantile)
+		if dir == dirIn {
+			out.InBPS = &v
+		} else {
+			out.OutBPS = &v
+		}
+	}
+	out.Coverage = min(coverage, 1)
+	return out, nil
+}
+
+// latest returns the latest of ts.
+func latest(ts ...time.Time) time.Time {
+	out := ts[0]
+	for _, t := range ts[1:] {
+		if t.After(out) {
+			out = t
+		}
+	}
+	return out
+}

internal/api/graph_plan.go

@@ -0,0 +1,119 @@
+package api
+
+import (
+	"errors"
+	"fmt"
+	"time"
+)
+
+const (
+	// maxGraphPoints bounds the points per series; the metric store caps a range query near 30,000.
+	maxGraphPoints = 11_000
+	// minRawStep is the default floor of a raw step: one poll interval.
+	minRawStep = 60 * time.Second
+	// targetPoints is how many points the default step aims for.
+	targetPoints = 1000
+	// longStepMin is the resolution of the long store's hourly rollup.
+	longStepMin = time.Hour
+)
+
+var errTooManyPoints = errors.New("too many points")
+
+// tierWindow is one stretch of a graph answered by one store: points at
+// from, from+step, ... up to and including last. bound is where the tier
+// officially ends (the join for the older tier of a stitched graph).
+type tierWindow struct {
+	tier  string
+	from  time.Time
+	last  time.Time
+	bound time.Time
+	step  time.Duration
+}
+
+// graphPlan says which stores answer which part of [from, to]. Tiers are oldest first.
+type graphPlan struct {
+	from, to   time.Time // aligned bounds of the whole graph
+	tiers      []tierWindow
+	stitchedAt *time.Time
+}
+
+func (p *graphPlan) newest() tierWindow { return p.tiers[len(p.tiers)-1] }
+
+// floorTo rounds t down to a multiple of d, counted from the Unix epoch so that
+// panning a graph never shifts its sample times.
+func floorTo(t time.Time, d time.Duration) time.Time {
+	ds := int64(d / time.Second)
+	return time.Unix(floorDiv(t.Unix(), ds)*ds, 0).UTC()
+}
+
+func ceilTo(t time.Time, d time.Duration) time.Time {
+	f := floorTo(t, d)
+	if f.Equal(t) {
+		return f
+	}
+	return f.Add(d)
+}
+
+func floorDiv(a, b int64) int64 {
+	q := a / b
+	if a%b != 0 && (a < 0) != (b < 0) {
+		q--
+	}
+	return q
+}
+
+// defaultStep is max(60 s, range/1000) rounded up to whole minutes, so the
+// same range always lands on the same grid.
+func defaultStep(rng time.Duration) time.Duration {
+	return max(minRawStep, ceilDur(rng/targetPoints, minRawStep))
+}
+
+func ceilDur(d, unit time.Duration) time.Duration {
+	return (d + unit - 1) / unit * unit
+}
+
+// planGraph decides the tiers for [from, to] at the given step (0: default).
+// hotStart is where the hot store's retention begins. A range that lies within
+// hot (give or take one step) is answered by raw data alone, a range wholly
+// older than hot by hourly data alone, and one that straddles hotStart is
+// stitched: hourly before the join, raw from it.
+func planGraph(from, to, hotStart time.Time, step time.Duration) (*graphPlan, error) {
+	if step <= 0 {
+		step = defaultStep(to.Sub(from))
+	}
+	if n := to.Sub(from) / step; n > maxGraphPoints {
+		return nil, fmt.Errorf("%w: %d points at a %s step; the limit is %d, use a larger step", errTooManyPoints, n, step, maxGraphPoints)
+	}
+	fromRaw, toRaw := floorTo(from, step), floorTo(to, step)
+	long := max(longStepMin, ceilDur(step, longStepMin))
+	switch {
+	case !fromRaw.Add(step).Before(hotStart):
+		return &graphPlan{from: fromRaw, to: toRaw, tiers: []tierWindow{
+			{tier: tierRaw, from: fromRaw, last: toRaw, bound: toRaw, step: step},
+		}}, nil
+	case !toRaw.After(hotStart):
+		fl, tl := floorTo(from, long), floorTo(to, long)
+		return &graphPlan{from: fl, to: tl, tiers: []tierWindow{
+			{tier: tier1h, from: fl, last: tl, bound: tl, step: long},
+		}}, nil
+	}
+	join := ceilTo(hotStart, step)
+	fl := floorTo(from, long)
+	return &graphPlan{from: fl, to: toRaw, stitchedAt: &join, tiers: []tierWindow{
+		{tier: tier1h, from: fl, last: floorTo(join.Add(-time.Nanosecond), long), bound: join, step: long},
+		{tier: tierRaw, from: join, last: toRaw, bound: toRaw, step: step},
+	}}, nil
+}
+
+// slots lists the sample times of a tier.
+func (w tierWindow) slots() []int64 {
+	if w.last.Before(w.from) {
+		return nil
+	}
+	n := int(w.last.Sub(w.from)/w.step) + 1
+	out := make([]int64, 0, n)
+	for t := w.from; !t.After(w.last); t = t.Add(w.step) {
+		out = append(out, t.Unix())
+	}
+	return out
+}

internal/api/graph_series.go

@@ -0,0 +1,80 @@
+package api
+
+import (
+	"math"
+	"sort"
+	"time"
+)
+
+// gapFactor is how many steps without data make a stretch "partial".
+const gapFactor = 2
+
+// slot is one sample time of the graph and the step of the tier it belongs to.
+type slot struct {
+	ts   int64
+	step int64
+}
+
+// gridOf lists every sample time of the plan, oldest first.
+func gridOf(p *graphPlan) []slot {
+	var out []slot
+	for _, w := range p.tiers {
+		step := int64(w.step / time.Second)
+		for _, ts := range w.slots() {
+			out = append(out, slot{ts: ts, step: step})
+		}
+	}
+	return out
+}
+
+// pointsOf lays vals over the grid; a slot without data is a null point.
+func pointsOf(grid []slot, vals map[int64]float64) []Point {
+	out := make([]Point, len(grid))
+	for i, g := range grid {
+		out[i] = Point{TS: g.ts}
+		if v, ok := vals[g.ts]; ok {
+			x := v
+			out[i].V = &x
+		}
+	}
+	return out
+}
+
+// findGaps returns the stretches of [start, end] longer than gapFactor steps in
+// which no slot has data. A stretch runs from the last slot with data (or the
+// start) to the next one (or the end); its step is the larger of its two
+// neighbours', so a join between tiers does not report a false gap.
+func findGaps(grid []slot, has func(ts int64) bool, start, end int64) []TimeSpan {
+	if len(grid) == 0 {
+		return nil
+	}
+	var gaps []TimeSpan
+	prevTS, prevStep := start, grid[0].step
+	report := func(from, to, step int64) {
+		if to-from > gapFactor*step {
+			gaps = append(gaps, TimeSpan{From: time.Unix(from, 0).UTC(), To: time.Unix(to, 0).UTC()})
+		}
+	}
+	for _, g := range grid {
+		if !has(g.ts) {
+			continue
+		}
+		report(prevTS, g.ts, max(prevStep, g.step))
+		prevTS, prevStep = g.ts, g.step
+	}
+	report(prevTS, end, prevStep)
+	return gaps
+}
+
+// nearestRank is the p-quantile (0 < p <= 1) of vals by the nearest-rank
+// method: the value at rank ceil(p*n) of the sorted list. It is exact, takes
+// an actual sample and needs no interpolation convention. NaN for no values.
+func nearestRank(vals []float64, p float64) float64 {
+	if len(vals) == 0 {
+		return math.NaN()
+	}
+	s := append([]float64(nil), vals...)
+	sort.Float64s(s)
+	rank := int(math.Ceil(p*float64(len(s)) - 1e-9)) // the epsilon keeps 0.95*20 from rounding up to 20
+	return s[min(max(rank, 1), len(s))-1]
+}

internal/api/graph_test.go

@@ -0,0 +1,444 @@
+package api_test
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"net/url"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/store"
+	"github.com/v10networks/nmon/internal/vmsup"
+)
+
+// graphVM fakes the two stores. hot and long say whether a store has a sample
+// at ts; the value is a function of direction and time so tests can check which store answered.
+type graphVM struct {
+	hot, long func(ts int64) bool
+	hotVal    func(dir string, ts int64) float64
+	longVal   func(dir string, ts int64) float64
+	longErr   error
+}
+
+func dirOf(expr string) string {
+	if strings.Contains(expr, `dir="in"`) || strings.Contains(expr, "InOctets") {
+		return "in"
+	}
+	return "out"
+}
+
+func (g *graphVM) fn(storeName, expr string, from, to time.Time, step time.Duration) ([]byte, error) {
+	has, val := g.hot, g.hotVal
+	if storeName == "long" {
+		if g.longErr != nil {
+			return nil, g.longErr
+		}
+		has, val = g.long, g.longVal
+	}
+	dir := dirOf(expr)
+	vals := map[int64]float64{}
+	for ts := from.Unix(); ts <= to.Unix(); ts += int64(step / time.Second) {
+		if has == nil || has(ts) {
+			vals[ts] = val(dir, ts)
+		}
+	}
+	if len(vals) == 0 {
+		return emptyMatrix, nil
+	}
+	return matrix(mrow{labels: map[string]string{"dir": dir}, vals: vals}), nil
+}
+
+func rising(dir string, ts int64) float64 {
+	if dir == "in" {
+		return 1e6 + float64(ts%100000)
+	}
+	return 2e6 + float64(ts%100000)
+}
+
+func newGraphEnv(t *testing.T, vm *graphVM, mut ...func(*store.Interface)) (*env, store.Interface) {
+	e := newEnv(t, 0)
+	e.q.fn = vm.fn
+	d := e.device(t, "Core", "", "")
+	return e, e.iface(t, d, 1, "xe-0/0/1", "uplink", mut...)
+}
+
+func iso(ts time.Time) string { return ts.UTC().Format(time.RFC3339) }
+
+func graphURL(id string, from, to time.Time, extra string) string {
+	return "/api/v1/graph?iface=" + id + "&from=" + url.QueryEscape(iso(from)) + "&to=" + url.QueryEscape(iso(to)) + extra
+}
+
+func TestGraphRawTierWithinHotRetention(t *testing.T) {
+	vm := &graphVM{hotVal: rising}
+	e, i := newGraphEnv(t, vm)
+	var g api.GraphResponse
+	rec := e.getJSON(t, graphURL(i.ID, t0.Add(-time.Hour), t0, ""), &g)
+	assertNoLeaks(t, "graph", rec.Body.Bytes())
+	if g.Tier != "raw" || g.StepSeconds != 60 || g.StitchedAt != nil || len(g.Tiers) != 0 || g.Unreliable {
+		t.Fatalf("raw graph: %+v", g)
+	}
+	if g.Partial == nil || len(g.Partial) != 0 {
+		t.Errorf("partial must be [] when complete: %v", g.Partial)
+	}
+	if g.Iface.ID != i.ID || g.Iface.IfName != "xe-0/0/1" || g.Iface.IfAlias != "uplink" || g.Iface.SpeedBPS != 1_000_000_000 || g.Iface.Device.Name != "Core" {
+		t.Errorf("iface: %+v", g.Iface)
+	}
+	if len(g.Series.In) != 61 || len(g.Series.Out) != 61 || g.Series.InMax != nil {
+		t.Fatalf("series: %d in, %d out", len(g.Series.In), len(g.Series.Out))
+	}
+	first, last := g.Series.In[0], g.Series.In[60]
+	if first.TS != t0.Add(-time.Hour).Unix() || last.TS != t0.Unix() || first.V == nil || *first.V != rising("in", first.TS) {
+		t.Errorf("first %+v last %+v", first, last)
+	}
+	if *g.Series.Out[10].V != rising("out", g.Series.Out[10].TS) {
+		t.Errorf("out series %+v", g.Series.Out[10])
+	}
+	qs := e.q.queries()
+	if len(qs) != 2 || qs[0].store != "hot" || qs[0].step != time.Minute ||
+		qs[0].expr != `rate(ifHCInOctets{iface="`+i.ID+`"}[120s])*8` || !strings.Contains(qs[1].expr, "ifHCOutOctets") {
+		t.Errorf("queries: %+v", qs)
+	}
+	var raw map[string]json.RawMessage
+	_ = json.Unmarshal(rec.Body.Bytes(), &raw)
+	for _, k := range []string{"stitched_at", "tiers", "unreliable_reason", "p95"} {
+		if _, ok := raw[k]; ok {
+			t.Errorf("%s must be omitted from a plain raw graph", k)
+		}
+	}
+	if !strings.HasPrefix(string(raw["series"]), `{"in":[[`) {
+		t.Errorf("points must be [ts,value] pairs: %.60s", raw["series"])
+	}
+}
+
+func TestGraphUses32BitCountersWhenNoHC(t *testing.T) {
+	e, i := newGraphEnv(t, &graphVM{hotVal: rising}, func(i *store.Interface) { i.HasHCCounters = false })
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, t0.Add(-time.Hour), t0, ""), &g)
+	if q := e.q.queries()[0].expr; !strings.Contains(q, "ifInOctets{") || strings.Contains(q, "HC") {
+		t.Errorf("32-bit query: %s", q)
+	}
+}
+
+func TestGraphStepDefaultsAndLimits(t *testing.T) {
+	for _, tc := range []struct {
+		rng  time.Duration
+		step int64
+	}{{time.Hour, 60}, {24 * time.Hour, 120}, {7 * 24 * time.Hour, 660}, {30 * 24 * time.Hour, 2640}} {
+		e, i := newGraphEnv(t, &graphVM{hotVal: rising})
+		var g api.GraphResponse
+		e.getJSON(t, graphURL(i.ID, t0.Add(-tc.rng), t0, ""), &g)
+		if g.StepSeconds != tc.step || g.Tier != "raw" || g.StitchedAt != nil {
+			t.Errorf("range %s: step %d tier %s stitched %v, want step %d raw", tc.rng, g.StepSeconds, g.Tier, g.StitchedAt, tc.step)
+		}
+		if n := len(g.Series.In); n < 700 && tc.rng >= 24*time.Hour || n > 1001 {
+			t.Errorf("range %s: %d points", tc.rng, n)
+		}
+		for _, p := range g.Series.In {
+			if p.TS%g.StepSeconds != 0 {
+				t.Fatalf("range %s: sample %d is not aligned to the %d s step", tc.rng, p.TS, g.StepSeconds)
+			}
+		}
+	}
+	e, i := newGraphEnv(t, &graphVM{hotVal: rising})
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, t0.Add(-time.Hour), t0, "&step=300"), &g)
+	if g.StepSeconds != 300 || len(g.Series.In) != 13 {
+		t.Errorf("explicit step: %d, %d points", g.StepSeconds, len(g.Series.In))
+	}
+	isProblem(t, e.do("GET", graphURL(i.ID, t0.Add(-30*24*time.Hour), t0, "&step=1"), ""), 422)
+}
+
+func TestGraphStitchesHourlyLongDataToRawHot(t *testing.T) {
+	join := t0.Add(-30 * 24 * time.Hour)
+	vm := &graphVM{
+		hot: func(ts int64) bool { return ts >= join.Unix() }, hotVal: rising,
+		longVal: func(dir string, ts int64) float64 { return -1 }, // long values are recognisable
+	}
+	e, i := newGraphEnv(t, vm)
+	from := t0.Add(-40 * 24 * time.Hour)
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, from, t0, "&max=true"), &g)
+	if g.StitchedAt == nil || g.Tier != "raw" || len(g.Tiers) != 2 {
+		t.Fatalf("stitched graph: stitched_at=%v tier=%s tiers=%+v", g.StitchedAt, g.Tier, g.Tiers)
+	}
+	if g.Tiers[0].Tier != "1h" || g.Tiers[0].StepSeconds != 3600 || g.Tiers[1].Tier != "raw" || g.Tiers[1].StepSeconds != g.StepSeconds {
+		t.Errorf("tiers: %+v step=%d", g.Tiers, g.StepSeconds)
+	}
+	if !g.Tiers[0].To.Equal(*g.StitchedAt) || !g.Tiers[1].From.Equal(*g.StitchedAt) || g.StitchedAt.Before(join) ||
+		g.StitchedAt.Sub(join) >= time.Duration(g.StepSeconds)*time.Second {
+		t.Errorf("stitched_at %v should be the hot retention edge %v rounded up by less than a step", g.StitchedAt, join)
+	}
+	var long, rawN int
+	prev := int64(0)
+	for _, p := range g.Series.In {
+		if p.TS <= prev {
+			t.Fatalf("series not strictly increasing at %d", p.TS)
+		}
+		prev = p.TS
+		if p.V == nil {
+			t.Fatalf("hole at %d: both stores have data here", p.TS)
+		}
+		if *p.V == -1 {
+			long++
+			if p.TS >= g.StitchedAt.Unix() || p.TS%3600 != 0 {
+				t.Errorf("long sample %d misplaced", p.TS)
+			}
+		} else {
+			rawN++
+			if p.TS < g.StitchedAt.Unix() {
+				t.Errorf("raw sample %d before the join", p.TS)
+			}
+		}
+	}
+	if long < 200 || rawN < 400 {
+		t.Errorf("%d long and %d raw samples", long, rawN)
+	}
+	if len(g.Partial) != 0 {
+		t.Errorf("no gaps expected: %v", g.Partial)
+	}
+	if len(g.Series.InMax) != len(g.Series.In) || len(g.Series.OutMax) != len(g.Series.Out) {
+		t.Errorf("max series lengths %d/%d", len(g.Series.InMax), len(g.Series.OutMax))
+	}
+	var sawLong, sawHot bool
+	for _, c := range e.q.queries() {
+		switch c.store {
+		case "long":
+			sawLong = true
+			if !strings.Contains(c.expr, "max by (dir)") || !strings.Contains(c.expr, `iface="`+i.ID+`"`) ||
+				(!strings.Contains(c.expr, "nmon_if_rate_bps_avg_1h") && !strings.Contains(c.expr, "nmon_if_rate_bps_max_1h")) {
+				t.Errorf("long query: %s", c.expr)
+			}
+		case "hot":
+			sawHot = true
+			if c.from.Before(*g.StitchedAt) {
+				t.Errorf("hot queried before the join: %v", c.from)
+			}
+		}
+	}
+	if !sawLong || !sawHot {
+		t.Error("a stitched graph must query both stores")
+	}
+}
+
+func TestGraphLongRangeUsesCoarserStepAndHourlyAverages(t *testing.T) {
+	e, i := newGraphEnv(t, &graphVM{hotVal: rising, longVal: rising})
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, t0.Add(-365*24*time.Hour), t0, ""), &g)
+	if g.StitchedAt == nil || g.Tiers[0].StepSeconds%3600 != 0 || g.Tiers[0].StepSeconds <= 3600 {
+		t.Fatalf("1y tiers: %+v", g.Tiers)
+	}
+	var longExpr string
+	for _, c := range e.q.queries() {
+		if c.store == "long" {
+			longExpr = c.expr
+		}
+	}
+	if !strings.Contains(longExpr, "avg_over_time((max by (dir)") || !strings.Contains(longExpr, ":3600s])") {
+		t.Errorf("coarse long query should average the hourly points: %s", longExpr)
+	}
+}
+
+func TestGraphOlderThanHotIsLongTierOnly(t *testing.T) {
+	vm := &graphVM{hot: func(int64) bool { return false }, hotVal: rising, longVal: rising}
+	e, i := newGraphEnv(t, vm)
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, t0.Add(-90*24*time.Hour), t0.Add(-60*24*time.Hour), ""), &g)
+	if g.Tier != "1h" || g.StepSeconds != 3600 || g.StitchedAt != nil || len(g.Tiers) != 0 || len(g.Series.In) < 700 {
+		t.Errorf("long-only graph: tier=%s step=%d stitched=%v %d points", g.Tier, g.StepSeconds, g.StitchedAt, len(g.Series.In))
+	}
+	for _, c := range e.q.queries() {
+		if c.store != "long" {
+			t.Errorf("hot queried for a range wholly beyond hot retention: %+v", c)
+		}
+	}
+}
+
+func TestGraphReportsPartialWhereNeitherStoreHasData(t *testing.T) {
+	// Until M5's rollup exists the long store is empty: the stitched span is partial, honestly.
+	join := t0.Add(-30 * 24 * time.Hour)
+	vm := &graphVM{hot: func(ts int64) bool { return ts >= join.Unix() }, hotVal: rising, long: func(int64) bool { return false }}
+	e, i := newGraphEnv(t, vm)
+	from := t0.Add(-40 * 24 * time.Hour)
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, from, t0, ""), &g)
+	if g.StitchedAt == nil || len(g.Partial) != 1 {
+		t.Fatalf("partial: %+v stitched %v", g.Partial, g.StitchedAt)
+	}
+	p := g.Partial[0]
+	if p.From.After(from.Add(time.Hour)) || p.To.Before(join) || p.To.After(join.Add(2*time.Hour)) {
+		t.Errorf("gap %v..%v, want about %v..%v", p.From, p.To, from, join)
+	}
+	for _, pt := range g.Series.In {
+		if pt.TS < g.StitchedAt.Unix() && pt.V != nil {
+			t.Fatalf("a null expected before the join, got %v at %d", *pt.V, pt.TS)
+		}
+	}
+
+	// a short hole in raw data is a partial span; a single missed sample is not
+	e2, i2 := newGraphEnv(t, &graphVM{hotVal: rising, hot: func(ts int64) bool {
+		m := (ts - t0.Add(-time.Hour).Unix()) / 60
+		return m < 20 || m > 50 && m != 55
+	}})
+	e2.getJSON(t, graphURL(i2.ID, t0.Add(-time.Hour), t0, ""), &g)
+	if len(g.Partial) != 1 || !g.Partial[0].From.Equal(t0.Add(-41*time.Minute)) || !g.Partial[0].To.Equal(t0.Add(-9*time.Minute)) {
+		t.Errorf("hole: %+v", g.Partial)
+	}
+
+	// no data at all: one partial span covering the range
+	e3, i3 := newGraphEnv(t, &graphVM{hotVal: rising, hot: func(int64) bool { return false }})
+	e3.getJSON(t, graphURL(i3.ID, t0.Add(-time.Hour), t0, ""), &g)
+	if len(g.Partial) != 1 || !g.Partial[0].From.Equal(t0.Add(-time.Hour)) || !g.Partial[0].To.Equal(t0) {
+		t.Errorf("empty: %+v", g.Partial)
+	}
+	// data that starts late: a leading gap
+	e4, i4 := newGraphEnv(t, &graphVM{hotVal: rising, hot: func(ts int64) bool { return ts >= t0.Add(-20*time.Minute).Unix() }})
+	e4.getJSON(t, graphURL(i4.ID, t0.Add(-time.Hour), t0, ""), &g)
+	if len(g.Partial) != 1 || !g.Partial[0].From.Equal(t0.Add(-time.Hour)) || !g.Partial[0].To.Equal(t0.Add(-20*time.Minute)) {
+		t.Errorf("leading gap: %+v", g.Partial)
+	}
+}
+
+func TestGraphLongStoreDownIsPartialNotAnError(t *testing.T) {
+	join := t0.Add(-30 * 24 * time.Hour)
+	vm := &graphVM{hot: func(ts int64) bool { return ts >= join.Unix() }, hotVal: rising, longErr: fmt.Errorf("%w: long", vmsup.ErrStoreDown)}
+	e, i := newGraphEnv(t, vm)
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, t0.Add(-40*24*time.Hour), t0, ""), &g)
+	if g.StitchedAt == nil || len(g.Partial) != 1 {
+		t.Errorf("long store down: %+v", g.Partial)
+	}
+}
+
+func TestGraphUnreliableInterfaceHasNoSeries(t *testing.T) {
+	for name, tc := range map[string]struct {
+		mut    func(*store.Interface)
+		reason string
+	}{
+		"32-bit fast link":  {func(i *store.Interface) { i.Unreliable, i.HasHCCounters = true, false }, "counter_32bit_fast_link"},
+		"discontinuities":   {func(i *store.Interface) { i.Unreliable, i.CounterDiscontinuity = true, 3 }, "counter_discontinuity"},
+		"unreliable for HC": {func(i *store.Interface) { i.Unreliable = true }, "counter_unreliable"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			e, i := newGraphEnv(t, &graphVM{hotVal: rising}, tc.mut)
+			var g api.GraphResponse
+			rec := e.getJSON(t, graphURL(i.ID, t0.Add(-time.Hour), t0, "&max=true&p95=true"), &g)
+			if !g.Unreliable || g.UnreliableReason != tc.reason || len(g.Series.In) != 0 || len(g.Series.Out) != 0 ||
+				g.P95 != nil || g.Partial == nil || g.Tier != "raw" || g.StepSeconds != 60 {
+				t.Errorf("unreliable graph: %+v", g)
+			}
+			if n := len(e.q.queries()); n != 0 {
+				t.Errorf("an unreliable interface must not be queried, %d queries", n)
+			}
+			if !strings.Contains(rec.Body.String(), `"in":[],"out":[]`) {
+				t.Errorf("series must be empty arrays: %s", rec.Body)
+			}
+		})
+	}
+}
+
+func TestGraphDirectionFilterAndMax(t *testing.T) {
+	e, i := newGraphEnv(t, &graphVM{hotVal: rising})
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, t0.Add(-time.Hour), t0, "&dir=out&max=true"), &g)
+	if len(g.Series.In) != 0 || len(g.Series.Out) != 61 || len(g.Series.OutMax) != 61 || g.Series.InMax != nil {
+		t.Errorf("dir=out: in=%d out=%d out_max=%d", len(g.Series.In), len(g.Series.Out), len(g.Series.OutMax))
+	}
+	qs := e.q.queries()
+	if len(qs) != 2 {
+		t.Fatalf("queries %d", len(qs))
+	}
+	for _, c := range qs {
+		if strings.Contains(c.expr, "In") {
+			t.Errorf("dir=out queried the in counter: %s", c.expr)
+		}
+	}
+	if !strings.Contains(qs[1].expr, "max_over_time((rate(ifHCOutOctets{iface=") || !strings.Contains(qs[1].expr, "[60s:60s])") {
+		t.Errorf("max query: %s", qs[1].expr)
+	}
+	isProblem(t, e.do("GET", graphURL(i.ID, t0.Add(-time.Hour), t0, "&dir=sideways"), ""), 422)
+}
+
+// referenceP95 is an independent nearest-rank implementation: the smallest
+// value such that at least 95% of the values are at or below it.
+func referenceP95(vals []float64) float64 {
+	s := append([]float64(nil), vals...)
+	sort.Float64s(s)
+	for _, v := range s {
+		n := 0
+		for _, x := range s {
+			if x <= v {
+				n++
+			}
+		}
+		if float64(n)*100 >= 95*float64(len(s)) {
+			return v
+		}
+	}
+	return math.NaN()
+}
+
+func TestGraphP95IsNearestRankOverFiveMinuteAverages(t *testing.T) {
+	// 24 h of 5-minute averages; one hour of the day has no data.
+	from := t0.Add(-24 * time.Hour)
+	vals := func(dir string, ts int64) float64 {
+		k := float64((ts / 300 * 7919) % 1000) // scrambled but deterministic
+		if dir == "out" {
+			k *= 3
+		}
+		return k * 1000
+	}
+	hole := func(ts int64) bool { return ts < t0.Add(-5*time.Hour).Unix() || ts >= t0.Add(-4*time.Hour).Unix() }
+	e, i := newGraphEnv(t, &graphVM{hotVal: vals, hot: hole})
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, from, t0, "&p95=true"), &g)
+	if g.P95 == nil || g.P95.Source != "live" || g.P95.InBPS == nil || g.P95.OutBPS == nil {
+		t.Fatalf("p95: %+v", g.P95)
+	}
+	var in, out []float64
+	for ts := from.Unix(); ts <= t0.Unix(); ts += 300 {
+		if hole(ts) {
+			in, out = append(in, vals("in", ts)), append(out, vals("out", ts))
+		}
+	}
+	if *g.P95.InBPS != referenceP95(in) || *g.P95.OutBPS != referenceP95(out) {
+		t.Errorf("p95 in %v out %v, reference %v / %v", *g.P95.InBPS, *g.P95.OutBPS, referenceP95(in), referenceP95(out))
+	}
+	wantCov := float64(len(in)) / (24 * 12)
+	if math.Abs(g.P95.Coverage-wantCov) > 0.01 || g.P95.Coverage >= 1 {
+		t.Errorf("coverage %v, want about %v", g.P95.Coverage, wantCov)
+	}
+	var withoutP95 api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, from, t0, ""), &withoutP95)
+	if withoutP95.P95 != nil {
+		t.Error("p95 only when asked")
+	}
+}
+
+func TestGraphErrors(t *testing.T) {
+	e, i := newGraphEnv(t, &graphVM{hotVal: rising})
+	isProblem(t, e.do("GET", "/api/v1/graph", ""), 422)
+	isProblem(t, e.do("GET", "/api/v1/graph?iface=01ARZ3NDEKTSV4RRFFQ69G5FAV", ""), 404)
+	isProblem(t, e.do("GET", graphURL(i.ID, t0, t0.Add(-time.Hour), ""), ""), 422)
+	isProblem(t, e.do("GET", "/api/v1/graph?iface="+i.ID+"&from=yesterday", ""), 422)
+	isProblem(t, e.do("GET", "/api/v1/graph?iface="+i.ID+"&step=0", ""), 422)
+	e.q.fn = func(string, string, time.Time, time.Time, time.Duration) ([]byte, error) {
+		return nil, fmt.Errorf("%w: hot", vmsup.ErrStoreDown)
+	}
+	isProblem(t, e.do("GET", graphURL(i.ID, t0.Add(-time.Hour), t0, ""), ""), 503)
+	e.q.fn = func(string, string, time.Time, time.Time, time.Duration) ([]byte, error) {
+		return nil, errors.New("boom")
+	}
+	isProblem(t, e.do("GET", graphURL(i.ID, t0.Add(-time.Hour), t0, ""), ""), 502)
+	// defaults: to=now, from=1h earlier
+	e.q.fn = (&graphVM{hotVal: rising}).fn
+	var g api.GraphResponse
+	e.getJSON(t, "/api/v1/graph?iface="+i.ID, &g)
+	if len(g.Series.In) != 61 || g.Series.In[60].TS != t0.Unix() {
+		t.Errorf("default range: %d points", len(g.Series.In))
+	}
+}

internal/api/graph_types.go

@@ -0,0 +1,125 @@
+package api
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+)
+
+// Graph tiers: raw samples from the hot store, or hourly rollups from the long store.
+const (
+	tierRaw = "raw"
+	tier1h  = "1h"
+)
+
+// Point is one sample of a series: Unix seconds and bits per second, or null
+// when the store had no data for that step. It marshals as [ts, value].
+type Point struct {
+	TS int64
+	V  *float64
+}
+
+// MarshalJSON writes the compact [unix_s, bps|null] form uPlot can use as is.
+func (p Point) MarshalJSON() ([]byte, error) {
+	return json.Marshal([]any{p.TS, p.V})
+}
+
+// UnmarshalJSON reads the [ts, value] form (tests and clients written in Go).
+func (p *Point) UnmarshalJSON(b []byte) error {
+	var raw [2]json.RawMessage
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return err
+	}
+	if err := json.Unmarshal(raw[0], &p.TS); err != nil {
+		return err
+	}
+	return json.Unmarshal(raw[1], &p.V)
+}
+
+// Schema describes Point as a two-element tuple. huma's Schema has no
+// prefixItems field, so it goes in as an extension; generated clients type it as [number, number | null].
+func (Point) Schema(r huma.Registry) *huma.Schema {
+	two := 2
+	r.Map()["Point"] = &huma.Schema{
+		Type: huma.TypeArray, MinItems: &two, MaxItems: &two,
+		Description: "[unix seconds, bits per second or null].",
+		Extensions: map[string]any{"prefixItems": []any{
+			map[string]any{"type": "number", "description": "unix seconds"},
+			map[string]any{"anyOf": []any{
+				map[string]any{"type": "number", "description": "bits per second"}, map[string]any{"type": "null"},
+			}},
+		}},
+	}
+	return &huma.Schema{Ref: "#/components/schemas/Point"}
+}
+
+// GraphIface identifies the graphed interface.
+type GraphIface struct {
+	ID       string    `json:"id"`
+	IfName   string    `json:"ifname"`
+	IfAlias  string    `json:"ifalias"`
+	SpeedBPS int64     `json:"speed_bps"`
+	Device   DeviceRef `json:"device"`
+}
+
+// TimeSpan is a half-open stretch of time.
+type TimeSpan struct {
+	From time.Time `json:"from"`
+	To   time.Time `json:"to"`
+}
+
+// GraphTier says which store and step answered one stretch of a stitched graph.
+type GraphTier struct {
+	From        time.Time `json:"from"`
+	To          time.Time `json:"to"`
+	Tier        string    `json:"tier" enum:"raw,1h"`
+	StepSeconds int64     `json:"step_seconds"`
+}
+
+// GraphSeries holds the bits-per-second series. in and out are always present
+// (empty for a direction not asked for); the max series only when requested.
+type GraphSeries struct {
+	In     []Point `json:"in"`
+	Out    []Point `json:"out"`
+	InMax  []Point `json:"in_max,omitempty"`
+	OutMax []Point `json:"out_max,omitempty"`
+}
+
+// set stores pts as the average (or maximum) series of dir.
+func (g *GraphSeries) set(dir string, isMax bool, pts []Point) {
+	switch {
+	case dir == dirIn && isMax:
+		g.InMax = pts
+	case dir == dirIn:
+		g.In = pts
+	case isMax:
+		g.OutMax = pts
+	default:
+		g.Out = pts
+	}
+}
+
+// GraphP95 is the 95th percentile of 5-minute averages (nearest rank).
+type GraphP95 struct {
+	InBPS    *float64 `json:"in_bps" nullable:"true"`
+	OutBPS   *float64 `json:"out_bps" nullable:"true"`
+	Source   string   `json:"source" enum:"materialised,live"`
+	Coverage float64  `json:"coverage" doc:"Share of the drawn range's 5-minute slots that had data, 0 to 1."`
+}
+
+// GraphResponse is a rate graph for one interface. A graph answered from two
+// stores always carries stitched_at; stretches with no data in either store are
+// listed in partial; an unreliable interface has no series at all.
+type GraphResponse struct {
+	Iface            GraphIface  `json:"iface"`
+	StepSeconds      int64       `json:"step_seconds" doc:"Step of the newest tier."`
+	Tier             string      `json:"tier" enum:"raw,1h" doc:"Tier of the newest data."`
+	StitchedAt       *time.Time  `json:"stitched_at,omitempty" doc:"Where the hourly long-store data ends and raw hot data begins."`
+	Tiers            []GraphTier `json:"tiers,omitempty" doc:"Present when two stores answered."`
+	UnreliableReason string      `json:"unreliable_reason,omitempty" doc:"Why the counters are not trusted, e.g. counter_32bit_fast_link."`
+	Partial          []TimeSpan  `json:"partial" doc:"Stretches of the range where neither store has data."`
+	Unreliable       bool        `json:"unreliable"`
+	Series           GraphSeries `json:"series"`
+	P95              *GraphP95   `json:"p95,omitempty"`
+}

internal/api/health.go

@@ -0,0 +1,184 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/metrics"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// PollerDeviceHealth is the poller's view of one device. It uses the slug, never the numeric ID.
+type PollerDeviceHealth struct {
+	Slug                string     `json:"slug"`
+	Name                string     `json:"name"`
+	State               string     `json:"state" enum:"ok,degraded,backoff" doc:"backoff: healthy but polled slower than its base interval."`
+	Varbinds            int64      `json:"varbinds" doc:"Varbinds per request right now."`
+	IntervalS           int64      `json:"interval_s" doc:"Current cadence, backoff included."`
+	BaseIntervalS       int64      `json:"base_interval_s"`
+	LastCycleS          float64    `json:"last_cycle_s" doc:"Duration of the last cycle."`
+	MissedDeadlines     int64      `json:"missed_deadlines" doc:"Consecutive cycles that ended at the deadline."`
+	ConsecutiveFailures int64      `json:"consecutive_failures"`
+	LastError           string     `json:"last_error"`
+	NextCycle           *time.Time `json:"next_cycle" nullable:"true"`
+	Profile             string     `json:"profile"`
+}
+
+// PollerTotals are the poller's aggregate numbers. The per-minute figures are
+// deltas of nmon's own counters over about the last minute.
+type PollerTotals struct {
+	PDUsPerMin           float64 `json:"pdus_per_min"`
+	TimeoutsPerMin       float64 `json:"timeouts_per_min"`
+	DeadlineMissesPerMin float64 `json:"deadline_misses_per_min"`
+	WriteQueueDepth      int64   `json:"write_queue_depth" doc:"Samples waiting in the hot write queue."`
+	WriteQueueCapacity   int64   `json:"write_queue_capacity"`
+	DevicesDegraded      int64   `json:"devices_degraded"`
+}
+
+// PollerHealth is the poller health page.
+type PollerHealth struct {
+	Devices []PollerDeviceHealth `json:"devices"`
+	Totals  PollerTotals         `json:"totals"`
+}
+
+func (s *server) registerHealth(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "get-poller-health", Method: http.MethodGet, Path: "/api/v1/poller/health", Tags: []string{"poller"},
+		Summary:     "Poller state of every polled device, with PDU, timeout and queue totals",
+		Description: "Rates are over about the last minute; the first call after start may report zero.",
+	}, s.pollerHealth)
+}
+
+func (s *server) pollerHealth(ctx context.Context, _ *struct{}) (*struct{ Body PollerHealth }, error) {
+	names, err := s.deviceNames(ctx)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	out := PollerHealth{Devices: []PollerDeviceHealth{}}
+	if s.PollerStatus != nil {
+		for _, d := range s.PollerStatus() {
+			h := PollerDeviceHealth{
+				Slug: d.Device, Name: names[d.Device], State: d.State, Varbinds: int64(d.Varbinds),
+				IntervalS: int64(d.Interval.Seconds()), BaseIntervalS: int64(d.BaseInterval.Seconds()),
+				LastCycleS: d.LastCycleDuration.Seconds(), MissedDeadlines: int64(d.MissedDeadlines),
+				ConsecutiveFailures: int64(d.ConsecutiveFailures), LastError: d.LastError, NextCycle: tp(d.NextCycle),
+				Profile: d.Profile,
+			}
+			if h.State == store.PollOK && d.Interval > d.BaseInterval {
+				h.State = store.PollBackoff
+			}
+			if h.State != store.PollOK {
+				out.Totals.DevicesDegraded++
+			}
+			out.Devices = append(out.Devices, h)
+		}
+	}
+	s.fillTotals(&out.Totals)
+	return &struct{ Body PollerHealth }{out}, nil
+}
+
+// deviceNames maps slugs to display names.
+func (s *server) deviceNames(ctx context.Context) (map[string]string, error) {
+	names := map[string]string{}
+	for off := int64(0); ; off += store.MaxLimit {
+		rows, n, err := s.Store.ListDevices(ctx, store.DeviceFilter{IncludeRetired: true}, store.Page{Offset: off, Limit: store.MaxLimit})
+		if err != nil {
+			return nil, err
+		}
+		for _, d := range rows {
+			names[d.Slug] = d.Name
+		}
+		if off+store.MaxLimit >= n {
+			return names, nil
+		}
+	}
+}
+
+// fillTotals reads nmon's own counters from the registry.
+func (s *server) fillTotals(t *PollerTotals) {
+	t.PDUsPerMin, t.TimeoutsPerMin, t.DeadlineMissesPerMin = s.rate.perMinute(s.readCounters())
+	for _, x := range s.Registry.Snapshot(s.Clock.Now()) {
+		if x.Name == "nmon_write_queue_depth" && labelValue(x.Labels, "queue") == "hot" {
+			t.WriteQueueDepth += int64(x.Value)
+		}
+	}
+	t.WriteQueueCapacity = int64(s.WriteQueueCapacity)
+}
+
+// counterSample is the registry's counter totals at one instant.
+type counterSample struct {
+	at                     time.Time
+	pdus, timeouts, misses float64
+}
+
+// readCounters sums the poll-path counters across all their series.
+func (s *server) readCounters() counterSample {
+	c := counterSample{at: s.Clock.Now()}
+	for _, x := range s.Registry.Snapshot(c.at) {
+		switch x.Name {
+		case "nmon_pdus_total":
+			if labelValue(x.Labels, "path") == "poll" {
+				c.pdus += x.Value
+			}
+		case "nmon_snmp_timeouts_total":
+			if labelValue(x.Labels, "path") == "poll" {
+				c.timeouts += x.Value
+			}
+		case "nmon_poll_deadline_missed_total":
+			c.misses += x.Value
+		}
+	}
+	return c
+}
+
+// counterRates turns successive counter totals into per-minute rates. It keeps
+// about a minute and a half of history and measures from the oldest sample, so
+// concurrent pollers of the page do not shrink the window.
+type counterRates struct {
+	mu      sync.Mutex
+	samples []counterSample
+}
+
+const rateHistory = 90 * time.Second
+
+// perMinute records cur and returns the per-minute rates since the oldest retained sample.
+func (c *counterRates) perMinute(cur counterSample) (pdus, timeouts, misses float64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for len(c.samples) > 0 && cur.at.Sub(c.samples[0].at) > rateHistory {
+		c.samples = c.samples[1:]
+	}
+	defer func() { c.samples = append(c.samples, cur) }()
+	if len(c.samples) == 0 {
+		return 0, 0, 0
+	}
+	base := c.samples[0]
+	span := cur.at.Sub(base.at).Minutes()
+	if span <= 0 {
+		return 0, 0, 0
+	}
+	per := func(now, then float64) float64 { return max(now-then, 0) / span }
+	return per(cur.pdus, base.pdus), per(cur.timeouts, base.timeouts), per(cur.misses, base.misses)
+}
+
+func labelValue(ls []metrics.Label, name string) string {
+	for _, l := range ls {
+		if l.Name == name {
+			return l.Value
+		}
+	}
+	return ""
+}
+
+// tp is t in UTC, or nil for the zero time.
+func tp(t time.Time) *time.Time {
+	if t.IsZero() {
+		return nil
+	}
+	u := t.UTC()
+	return &u
+}

internal/api/helpers_test.go

@@ -0,0 +1,180 @@
+package api_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/store"
+	"github.com/v10networks/nmon/internal/ulid"
+)
+
+// device stores a network device and returns it with its generated slug.
+func (e *env) device(t *testing.T, name, site, role string, mut ...func(*store.Device)) *store.Device {
+	t.Helper()
+	d := &store.Device{Name: name, Kind: store.KindNetwork, Site: site, Role: role, Enabled: true, Profile: "generic", MgmtIP: "192.0.2.1"}
+	for _, m := range mut {
+		m(d)
+	}
+	if err := e.st.CreateDevice(context.Background(), d); err != nil {
+		t.Fatal(err)
+	}
+	return d
+}
+
+// iface stores one interface of d and returns it.
+func (e *env) iface(t *testing.T, d *store.Device, idx int64, name, alias string, mut ...func(*store.Interface)) store.Interface {
+	t.Helper()
+	i := store.Interface{
+		ID: ulid.New(e.clk.Now()), DeviceID: d.ID, IfIndex: idx, IfName: name, IfDescr: name, IfAlias: alias,
+		IfSpeed: 1_000_000_000, IfHighSpeed: 1000, AdminStatus: 1, OperStatus: 1, HasHCCounters: true,
+		FirstSeen: e.clk.Now(), LastDiscovered: e.clk.Now(),
+	}
+	for _, m := range mut {
+		m(&i)
+	}
+	if err := e.st.UpsertInterfaces(context.Background(), d.ID, []store.Interface{i}); err != nil {
+		t.Fatal(err)
+	}
+	return i
+}
+
+// getJSON GETs path and decodes a 200 body into v.
+func (e *env) getJSON(t *testing.T, path string, v any) *httptest.ResponseRecorder {
+	t.Helper()
+	rec := e.do("GET", path, "")
+	if rec.Code != 200 {
+		t.Fatalf("GET %s: %d %s", path, rec.Code, rec.Body)
+	}
+	if err := json.Unmarshal(rec.Body.Bytes(), v); err != nil {
+		t.Fatalf("GET %s: %v: %s", path, err, rec.Body)
+	}
+	return rec
+}
+
+// vector builds an instant-query response from rows of (labels, value).
+type vrow struct {
+	labels map[string]string
+	v      float64
+}
+
+func vector(rows ...vrow) []byte {
+	var b bytes.Buffer
+	b.WriteString(`{"status":"success","data":{"resultType":"vector","result":[`)
+	for i, r := range rows {
+		if i > 0 {
+			b.WriteByte(',')
+		}
+		lb, _ := json.Marshal(r.labels)
+		fmt.Fprintf(&b, `{"metric":%s,"value":[1,"%v"]}`, lb, r.v)
+	}
+	b.WriteString(`]}}`)
+	return b.Bytes()
+}
+
+// matrix builds a range-query response with one series per entry of rows:
+// labels and a map of unix second to value.
+type mrow struct {
+	labels map[string]string
+	vals   map[int64]float64
+}
+
+func matrix(rows ...mrow) []byte {
+	var b bytes.Buffer
+	b.WriteString(`{"status":"success","data":{"resultType":"matrix","result":[`)
+	for i, r := range rows {
+		if i > 0 {
+			b.WriteByte(',')
+		}
+		lb, _ := json.Marshal(r.labels)
+		fmt.Fprintf(&b, `{"metric":%s,"values":[`, lb)
+		first := true
+		for _, ts := range sortedKeys(r.vals) {
+			if !first {
+				b.WriteByte(',')
+			}
+			first = false
+			fmt.Fprintf(&b, `[%d,"%v"]`, ts, r.vals[ts])
+		}
+		b.WriteString(`]}`)
+	}
+	b.WriteString(`]}}`)
+	return b.Bytes()
+}
+
+func sortedKeys(m map[int64]float64) []int64 {
+	out := make([]int64, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	for i := 1; i < len(out); i++ {
+		for j := i; j > 0 && out[j] < out[j-1]; j-- {
+			out[j], out[j-1] = out[j-1], out[j]
+		}
+	}
+	return out
+}
+
+// emptyMatrix is a range response with no series.
+var emptyMatrix = []byte(`{"status":"success","data":{"resultType":"matrix","result":[]}}`)
+
+// walkJSON calls fn for every key/value of every object in the JSON document.
+func walkJSON(t *testing.T, body []byte, fn func(path, key string, val any)) {
+	t.Helper()
+	var doc any
+	if err := json.Unmarshal(body, &doc); err != nil {
+		t.Fatalf("not JSON: %v: %s", err, body)
+	}
+	var rec func(path string, v any)
+	rec = func(path string, v any) {
+		switch x := v.(type) {
+		case map[string]any:
+			for k, c := range x {
+				fn(path, k, c)
+				rec(path+"."+k, c)
+			}
+		case []any:
+			for i, c := range x {
+				rec(fmt.Sprintf("%s[%d]", path, i), c)
+			}
+		}
+	}
+	rec("$", doc)
+}
+
+// assertNoLeaks fails when body has a numeric id, a credential field or a secret.
+func assertNoLeaks(t *testing.T, what string, body []byte, secrets ...string) {
+	t.Helper()
+	walkJSON(t, body, func(path, key string, val any) {
+		lower := strings.ToLower(key)
+		if _, isNum := val.(float64); isNum && (lower == "id" || strings.HasSuffix(lower, "_id") || lower == "device_id") {
+			t.Errorf("%s: numeric id at %s.%s = %v", what, path, key, val)
+		}
+		for _, bad := range []string{"community", "password", "pass", "hash", "secret", "token", "credential"} {
+			if strings.Contains(lower, bad) {
+				t.Errorf("%s: credential-like key %s.%s", what, path, key)
+			}
+		}
+	})
+	for _, s := range secrets {
+		if bytes.Contains(body, []byte(s)) {
+			t.Errorf("%s: response contains secret %q", what, s)
+		}
+	}
+}
+
+var t0 = time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
+
+// doFrom is do with a chosen remote address, which the admission limits key on.
+func (e *env) doFrom(method, path, remote string) *httptest.ResponseRecorder {
+	req := httptest.NewRequest(method, path, nil)
+	req.RemoteAddr = remote
+	rec := httptest.NewRecorder()
+	e.h.ServeHTTP(rec, req)
+	return rec
+}

internal/api/integration_test.go

@@ -0,0 +1,208 @@
+//go:build integration && linux
+
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/metrics"
+	"github.com/v10networks/nmon/internal/store"
+	"github.com/v10networks/nmon/internal/testutil"
+	"github.com/v10networks/nmon/internal/ulid"
+	"github.com/v10networks/nmon/internal/vmsup"
+)
+
+// TestIntegrationGraphFromRealStores writes synthetic octet counters whose rate
+// rises, and hourly rollups for the weeks before, into real hot and long
+// stores, then asks the API for a one-hour graph and a stitched 40-day graph.
+func TestIntegrationGraphFromRealStores(t *testing.T) {
+	bin, err := filepath.Abs(filepath.Join("..", "vmsup", "bin", "victoria-metrics-prod-linux-"+runtime.GOARCH))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(bin); err != nil {
+		t.Skipf("store binary not found (%v); run scripts/fetch-vm.sh", err)
+	}
+	t.Setenv("NMON_VM_BINARY", bin)
+	t.Setenv("NMON_VM_BINARY_UNVERIFIED", "")
+	sup, err := vmsup.New(vmsup.Options{DataDir: t.TempDir()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+	if err := sup.Start(ctx); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = sup.Stop(context.Background()) })
+
+	clk := clock.Real()
+	st := testutil.OpenStore(t, clk)
+	dev := &store.Device{Name: "Core", Kind: store.KindNetwork, Enabled: true, Profile: "generic"}
+	if err := st.CreateDevice(ctx, dev); err != nil {
+		t.Fatal(err)
+	}
+	ifc := store.Interface{ID: ulid.New(clk.Now()), DeviceID: dev.ID, IfIndex: 1, IfName: "xe-0/0/1", IfSpeed: 1e9, IfHighSpeed: 1000,
+		AdminStatus: 1, OperStatus: 1, HasHCCounters: true}
+	if err := st.UpsertInterfaces(ctx, dev.ID, []store.Interface{ifc}); err != nil {
+		t.Fatal(err)
+	}
+
+	// 3 h of one-minute samples ending 2 min ago. The in rate climbs from 1 to 4 Mbit/s; out is a flat 500 kbit/s.
+	end := time.Now().Add(-2 * time.Minute).Truncate(time.Minute)
+	const minutes = 3 * 60
+	var hot []metrics.Sample
+	var in, out float64
+	for i := 0; i <= minutes; i++ {
+		ts := end.Add(-time.Duration(minutes-i) * time.Minute)
+		inBps := 1e6 + 3e6*float64(i)/minutes
+		in += inBps / 8 * 60
+		out += 500e3 / 8 * 60
+		hot = append(hot,
+			metrics.New("ifHCInOctets", in, ts, "device", dev.Slug, "iface", ifc.ID),
+			metrics.New("ifHCOutOctets", out, ts, "device", dev.Slug, "iface", ifc.ID))
+	}
+	// Hourly rollups for 40 days back to 31 days ago, 800 kbit/s in and 200 out.
+	var long []metrics.Sample
+	for h := 40 * 24; h >= 31*24; h-- {
+		ts := time.Now().Add(-time.Duration(h) * time.Hour).Truncate(time.Hour)
+		long = append(long,
+			metrics.New("nmon_if_rate_bps_avg_1h", 800e3, ts, "device", dev.Slug, "iface", ifc.ID, "dir", "in", "v", "1"),
+			metrics.New("nmon_if_rate_bps_avg_1h", 200e3, ts, "device", dev.Slug, "iface", ifc.ID, "dir", "out", "v", "1"))
+	}
+	for name, ss := range map[string][]metrics.Sample{vmsup.StoreHot: hot, vmsup.StoreLong: long} {
+		q := vmsup.NewQueue(name, sup, vmsup.QueueOptions{MaxAge: 100 * 24 * time.Hour})
+		if err := q.Write(ctx, ss); err != nil {
+			t.Fatal(err)
+		}
+		if err := q.Flush(ctx); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	h := api.New(api.Deps{Store: st, Querier: vmsup.NewQuerier(sup), Poller: noRefresh{}, Clock: clk})
+	get := func(q url.Values) api.GraphResponse {
+		rec := httptest.NewRecorder()
+		h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/graph?"+q.Encode(), nil))
+		if rec.Code != http.StatusOK {
+			t.Fatalf("graph: %d %s", rec.Code, rec.Body)
+		}
+		var g api.GraphResponse
+		if err := json.Unmarshal(rec.Body.Bytes(), &g); err != nil {
+			t.Fatal(err)
+		}
+		return g
+	}
+	// The store needs a moment to make flushed samples searchable.
+	var g api.GraphResponse
+	deadline := time.Now().Add(30 * time.Second)
+	for {
+		now := time.Now()
+		g = get(url.Values{"iface": {ifc.ID}, "from": {now.Add(-time.Hour).UTC().Format(time.RFC3339)}, "to": {now.UTC().Format(time.RFC3339)},
+			"max": {"true"}, "p95": {"true"}})
+		if countValues(g.Series.In) > 30 || time.Now().After(deadline) {
+			break
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	if g.Tier != "raw" || g.StitchedAt != nil || g.StepSeconds != 60 {
+		t.Fatalf("1h graph: tier=%s step=%d stitched=%v", g.Tier, g.StepSeconds, g.StitchedAt)
+	}
+	first, last := firstLast(g.Series.In)
+	if first == nil || last == nil || *last <= *first*1.1 || *first < 2.5e6 || *last > 4.2e6 {
+		t.Errorf("in series should rise from about 2.5 to 4 Mbit/s: first %v last %v", deref(first), deref(last))
+	}
+	if f, l := firstLast(g.Series.Out); f == nil || *f < 450e3 || *f > 550e3 || *l < 450e3 || *l > 550e3 {
+		t.Errorf("out series should be a flat 500 kbit/s: %v %v", deref(f), deref(l))
+	}
+	if countValues(g.Series.InMax) < 30 {
+		t.Errorf("max series: %d values", countValues(g.Series.InMax))
+	}
+	if g.P95 == nil || g.P95.InBPS == nil || *g.P95.InBPS < 2.5e6 || g.P95.Source != "live" || g.P95.Coverage <= 0 {
+		t.Errorf("p95: %+v", g.P95)
+	}
+
+	now := time.Now()
+	g = get(url.Values{"iface": {ifc.ID}, "from": {now.Add(-40 * 24 * time.Hour).UTC().Format(time.RFC3339)}, "to": {now.UTC().Format(time.RFC3339)}})
+	if g.StitchedAt == nil || len(g.Tiers) != 2 || g.Tiers[0].Tier != "1h" {
+		t.Fatalf("40d graph should be stitched: %+v", g.Tiers)
+	}
+	var fromLong int
+	for _, p := range g.Series.In {
+		if p.TS < g.StitchedAt.Unix() && p.V != nil {
+			fromLong++
+			if *p.V != 800e3 {
+				t.Fatalf("long sample %d = %v, want 800000", p.TS, *p.V)
+			}
+		}
+	}
+	if fromLong < 150 {
+		t.Errorf("only %d hourly samples came from the long store", fromLong)
+	}
+	if len(g.Partial) == 0 {
+		t.Error("hot has 3 h of data in a 30-day window: the rest is partial")
+	}
+
+	// A year needs a coarser step: the store must accept the averaging subquery and average equal hours to the same value.
+	g = get(url.Values{"iface": {ifc.ID}, "from": {now.Add(-365 * 24 * time.Hour).UTC().Format(time.RFC3339)}, "to": {now.UTC().Format(time.RFC3339)}})
+	if g.StitchedAt == nil || g.Tiers[0].StepSeconds <= 3600 {
+		t.Fatalf("1y tiers: %+v", g.Tiers)
+	}
+	var coarse int
+	for _, p := range g.Series.In {
+		if p.TS < g.StitchedAt.Unix() && p.V != nil {
+			coarse++
+			if *p.V < 799e3 || *p.V > 801e3 {
+				t.Errorf("coarse long sample %d = %v, want 800000", p.TS, *p.V)
+			}
+		}
+	}
+	if coarse == 0 {
+		t.Error("the 1y graph has no long-store samples")
+	}
+}
+
+type noRefresh struct{}
+
+func (noRefresh) Refresh() {}
+
+func countValues(ps []api.Point) int {
+	n := 0
+	for _, p := range ps {
+		if p.V != nil {
+			n++
+		}
+	}
+	return n
+}
+
+func firstLast(ps []api.Point) (first, last *float64) {
+	for _, p := range ps {
+		if p.V == nil {
+			continue
+		}
+		if first == nil {
+			first = p.V
+		}
+		last = p.V
+	}
+	return first, last
+}
+
+func deref(p *float64) float64 {
+	if p == nil {
+		return -1
+	}
+	return *p
+}

internal/api/interfaces.go

@@ -0,0 +1,78 @@
+package api
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// InterfaceHit is one interface search result with its device and live rates.
+type InterfaceHit struct {
+	ID          string    `json:"id" doc:"Interface ULID."`
+	Device      DeviceRef `json:"device"`
+	IfName      string    `json:"ifname"`
+	IfAlias     string    `json:"ifalias"`
+	IfDescr     string    `json:"ifdescr"`
+	OperStatus  int64     `json:"oper_status"`
+	AdminStatus int64     `json:"admin_status"`
+	SpeedBPS    int64     `json:"speed_bps" doc:"Link speed in bits per second (ifHighSpeed when set)."`
+	InBPS       *float64  `json:"in_bps" nullable:"true"`
+	OutBPS      *float64  `json:"out_bps" nullable:"true"`
+}
+
+// InterfaceList is a page of InterfaceHit.
+type InterfaceList struct {
+	Items   []InterfaceHit `json:"items"`
+	Page    int64          `json:"page"`
+	PerPage int64          `json:"per_page"`
+	Total   int64          `json:"total"`
+}
+
+func (s *server) registerInterfaces(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "list-interfaces", Method: http.MethodGet, Path: "/api/v1/interfaces", Tags: []string{"interfaces"},
+		Summary:     "Search interfaces by name or alias",
+		Description: "q matches ifname and ifalias, case-insensitively, across all devices. Rates are null when unknown or unreliable.",
+	}, s.listInterfaces)
+}
+
+func (s *server) listInterfaces(ctx context.Context, in *struct {
+	Q string `query:"q" doc:"Substring of ifname or ifalias."`
+	PageParams
+}) (*struct{ Body InterfaceList }, error) {
+	rows, total, err := s.Store.SearchInterfaceHits(ctx, in.Q, store.Page{Offset: in.offset(), Limit: in.limit()})
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	out := InterfaceList{Items: make([]InterfaceHit, 0, len(rows)), Page: in.pageNo(), PerPage: in.limit(), Total: total}
+	var rates map[string]*ifRates
+	if len(rows) > 0 {
+		ids := make([]string, len(rows))
+		for i, r := range rows {
+			ids[i] = r.ID
+		}
+		rates, _ = s.liveRates(ctx, `iface=~`+quote(regexAlt(ids)), false)
+	}
+	for i := range rows {
+		r := &rows[i]
+		inB, outB := rates[r.ID].bps(&r.Interface)
+		out.Items = append(out.Items, InterfaceHit{
+			ID: r.ID, Device: DeviceRef{Slug: r.DeviceSlug, Name: r.DeviceName}, IfName: r.IfName, IfAlias: r.IfAlias,
+			IfDescr: r.IfDescr, OperStatus: r.OperStatus, AdminStatus: r.AdminStatus, SpeedBPS: speedBPS(&r.Interface),
+			InBPS: inB, OutBPS: outB,
+		})
+	}
+	return &struct{ Body InterfaceList }{out}, nil
+}
+
+// speedBPS is the link speed in bits per second: ifHighSpeed (Mbit/s) when the
+// device reports it, since ifSpeed saturates at 4.29 Gbit/s.
+func speedBPS(i *store.Interface) int64 {
+	if i.IfHighSpeed > 0 {
+		return i.IfHighSpeed * 1_000_000
+	}
+	return i.IfSpeed
+}

internal/api/list_test.go

@@ -0,0 +1,206 @@
+package api_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+func TestDeviceListShapeFiltersAndPaging(t *testing.T) {
+	e := newEnv(t, 0)
+	a := e.device(t, "Core A", "yyz", "core")
+	b := e.device(t, "Edge B", "yul", "edge")
+	e.device(t, "Edge C", "yul", "edge", func(d *store.Device) { d.Kind = store.KindVM })
+	e.iface(t, a, 1, "ge-0/0/0", "uplink")
+	e.iface(t, a, 2, "ge-0/0/1", "", func(i *store.Interface) { i.OperStatus = 2 }) // admin up, oper down
+	e.iface(t, a, 3, "ge-0/0/2", "", func(i *store.Interface) { i.AdminStatus, i.OperStatus = 2, 2 })
+	e.iface(t, a, 4, "mgmt", "", func(i *store.Interface) { i.Excluded = true })
+	if err := e.st.TouchDevice(t.Context(), a.ID, e.clk.Now().Add(-10*time.Second)); err != nil {
+		t.Fatal(err)
+	}
+	if err := e.st.SetPollState(t.Context(), b.ID, store.PollDegraded, []string{"slow"}); err != nil {
+		t.Fatal(err)
+	}
+
+	var l api.DeviceList
+	rec := e.getJSON(t, "/api/v1/devices", &l)
+	if l.Total != 3 || l.Page != 1 || l.PerPage != 100 || len(l.Items) != 3 {
+		t.Fatalf("envelope: %+v", l)
+	}
+	assertNoLeaks(t, "device list", rec.Body.Bytes())
+	got := map[string]api.DeviceSummary{}
+	for _, d := range l.Items {
+		got[d.Name] = d
+	}
+	core := got["Core A"]
+	if core.InterfacesTotal != 3 || core.InterfacesDown != 1 || core.Reachability != api.ReachUp || core.PollState != "ok" ||
+		core.LastSeen == nil || core.Site != "yyz" || core.Warnings == nil || core.Profile != "generic" || core.MgmtIP == "" {
+		t.Errorf("core summary: %+v", core)
+	}
+	if edge := got["Edge B"]; edge.PollState != "degraded" || len(edge.Warnings) != 1 || edge.Reachability != api.ReachUnknown {
+		t.Errorf("edge summary: %+v", edge)
+	}
+
+	for q, want := range map[string]int{"site=yul": 2, "role=core": 1, "kind=vm": 1, "q=edge": 2, "q=EDGE+b": 1, "site=yul&role=core": 0} {
+		var f api.DeviceList
+		e.getJSON(t, "/api/v1/devices?"+q, &f)
+		if len(f.Items) != want || f.Total != int64(want) {
+			t.Errorf("%s: %d items, total %d, want %d", q, len(f.Items), f.Total, want)
+		}
+	}
+	var p api.DeviceList
+	e.getJSON(t, "/api/v1/devices?per_page=2&page=2", &p)
+	if len(p.Items) != 1 || p.Total != 3 || p.Page != 2 || p.PerPage != 2 {
+		t.Errorf("page 2: %+v", p)
+	}
+	e.getJSON(t, "/api/v1/devices?state=up", &p)
+	if p.Total != 1 || p.Items[0].Name != "Core A" {
+		t.Errorf("state=up: %+v", p)
+	}
+	e.getJSON(t, "/api/v1/devices?state=unknown&per_page=1&page=2", &p)
+	if p.Total != 2 || len(p.Items) != 1 {
+		t.Errorf("state=unknown page 2: %+v", p)
+	}
+}
+
+func TestListLimitsAreProblems(t *testing.T) {
+	e := newEnv(t, 0)
+	e.getJSON(t, "/api/v1/devices?per_page=1000", new(api.DeviceList))
+	for _, path := range []string{
+		"/api/v1/devices?per_page=1001", "/api/v1/devices?per_page=0", "/api/v1/devices?page=0",
+		"/api/v1/devices?kind=toaster", "/api/v1/devices?state=sideways", "/api/v1/interfaces?per_page=5000",
+		"/api/v1/identity-reviews?per_page=1001",
+	} {
+		isProblem(t, e.do("GET", path, ""), 422)
+	}
+}
+
+func TestDeviceFacets(t *testing.T) {
+	e := newEnv(t, 0)
+	e.device(t, "A", "yyz", "core")
+	e.device(t, "B", "yul", "edge")
+	e.device(t, "C", "", "", func(d *store.Device) { d.Kind = store.KindVM })
+	gone := e.device(t, "D", "sea", "lab")
+	if err := e.st.RetireDevice(t.Context(), gone.ID); err != nil {
+		t.Fatal(err)
+	}
+	var f api.DeviceFacets
+	e.getJSON(t, "/api/v1/devices/facets", &f)
+	if strings.Join(f.Sites, ",") != "yul,yyz" || strings.Join(f.Roles, ",") != "core,edge" || strings.Join(f.Kinds, ",") != "network,vm" {
+		t.Errorf("facets: %+v", f)
+	}
+	var empty api.DeviceFacets
+	newEnv(t, 0).getJSON(t, "/api/v1/devices/facets", &empty)
+	if empty.Sites == nil || empty.Roles == nil || empty.Kinds == nil {
+		t.Errorf("empty facets must be [] not null: %+v", empty)
+	}
+}
+
+func TestReachabilityFromUpSeriesAndLastSeen(t *testing.T) {
+	e := newEnv(t, 0)
+	up := e.device(t, "Up", "", "")
+	half := e.device(t, "Half", "", "")
+	down := e.device(t, "Down", "", "")
+	stale := e.device(t, "Stale", "", "")
+	maint := e.device(t, "Maint", "", "", func(d *store.Device) { d.PollState = store.PollMaintenance })
+	off := e.device(t, "Off", "", "", func(d *store.Device) { d.Enabled = false })
+	for _, d := range []*store.Device{up, half, down, maint, off} {
+		_ = e.st.TouchDevice(t.Context(), d.ID, e.clk.Now())
+	}
+	_ = e.st.TouchDevice(t.Context(), stale.ID, e.clk.Now().Add(-time.Hour))
+	e.q.fn = func(_, expr string, _, _ time.Time, _ time.Duration) ([]byte, error) {
+		switch {
+		case strings.Contains(expr, "nmon_icmp_up"):
+			return vector(
+				vrow{map[string]string{"device": up.Slug}, 1}, vrow{map[string]string{"device": half.Slug}, 1},
+				vrow{map[string]string{"device": down.Slug}, 0}), nil
+		case strings.Contains(expr, "nmon_snmp_up"):
+			return vector(
+				vrow{map[string]string{"device": up.Slug}, 1}, vrow{map[string]string{"device": half.Slug}, 0},
+				vrow{map[string]string{"device": down.Slug}, 0}), nil
+		}
+		return emptyMatrix, nil
+	}
+	var l api.DeviceList
+	e.getJSON(t, "/api/v1/devices", &l)
+	got := map[string]string{}
+	for _, d := range l.Items {
+		got[d.Name] = d.Reachability
+	}
+	want := map[string]string{"Up": "up", "Half": "warning", "Down": "down", "Stale": "down", "Maint": "maintenance", "Off": "unknown"}
+	for n, w := range want {
+		if got[n] != w {
+			t.Errorf("%s: %q, want %q (all: %v)", n, got[n], w, got)
+		}
+	}
+	// a store that is down falls back to last_seen
+	e.q.fn = func(string, string, time.Time, time.Time, time.Duration) ([]byte, error) {
+		return nil, fmt.Errorf("down")
+	}
+	e.getJSON(t, "/api/v1/devices", &l)
+	for _, d := range l.Items {
+		if d.Name == "Up" && d.Reachability != "up" || d.Name == "Stale" && d.Reachability != "down" {
+			t.Errorf("fallback: %s is %s", d.Name, d.Reachability)
+		}
+	}
+}
+
+func TestInterfaceSearch(t *testing.T) {
+	e := newEnv(t, 0)
+	a := e.device(t, "Core A", "", "")
+	b := e.device(t, "Edge B", "", "")
+	up := e.iface(t, a, 1, "xe-0/0/1", "Uplink to ISP")
+	e.iface(t, a, 2, "xe-0/0/2", "customer")
+	e.iface(t, b, 1, "Gi1", "UPLINK backup")
+	hc32 := e.iface(t, b, 2, "Gi2", "uplink-32", func(i *store.Interface) { i.HasHCCounters = false })
+	e.iface(t, b, 3, "Gi3", "uplink-bad", func(i *store.Interface) { i.Unreliable = true })
+	gone := e.iface(t, b, 4, "Gi4", "uplink-gone")
+	if err := e.st.RetireInterface(t.Context(), gone.ID); err != nil {
+		t.Fatal(err)
+	}
+	e.q.fn = func(_, expr string, _, _ time.Time, _ time.Duration) ([]byte, error) {
+		if !strings.Contains(expr, "ifHCInOctets") {
+			return emptyMatrix, nil
+		}
+		return vector(
+			vrow{map[string]string{"iface": up.ID, "k": "in_hc"}, 1000}, vrow{map[string]string{"iface": up.ID, "k": "out_hc"}, 2000},
+			vrow{map[string]string{"iface": hc32.ID, "k": "in_32"}, 31}, vrow{map[string]string{"iface": hc32.ID, "k": "out_32"}, 32},
+		), nil
+	}
+	var l api.InterfaceList
+	rec := e.getJSON(t, "/api/v1/interfaces?q=UPLINK", &l)
+	assertNoLeaks(t, "interface search", rec.Body.Bytes())
+	if l.Total != 4 || len(l.Items) != 4 {
+		t.Fatalf("hits: %+v", l)
+	}
+	byAlias := map[string]api.InterfaceHit{}
+	for _, h := range l.Items {
+		byAlias[h.IfAlias] = h
+	}
+	h := byAlias["Uplink to ISP"]
+	if h.ID != up.ID || h.Device.Slug != a.Slug || h.Device.Name != "Core A" || h.SpeedBPS != 1_000_000_000 ||
+		h.InBPS == nil || *h.InBPS != 1000 || *h.OutBPS != 2000 || h.OperStatus != 1 || h.AdminStatus != 1 {
+		t.Errorf("hit: %+v", h)
+	}
+	if x := byAlias["uplink-32"]; x.InBPS == nil || *x.InBPS != 31 {
+		t.Errorf("32-bit interface should use ifInOctets rates: %+v", x)
+	}
+	if x := byAlias["uplink-bad"]; x.InBPS != nil || x.OutBPS != nil {
+		t.Errorf("unreliable interface must have null rates: %+v", x)
+	}
+	if x := byAlias["UPLINK backup"]; x.InBPS != nil {
+		t.Errorf("no data means null: %+v", x)
+	}
+	e.getJSON(t, "/api/v1/interfaces?q=xe-0&per_page=1&page=2", &l)
+	if l.Total != 2 || len(l.Items) != 1 || l.Page != 2 {
+		t.Errorf("paging: %+v", l)
+	}
+	e.getJSON(t, "/api/v1/interfaces?q=zzz", &l)
+	if l.Total != 0 || l.Items == nil {
+		t.Errorf("no hits: %+v", l)
+	}
+}

internal/api/meta.go

@@ -0,0 +1,41 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/buildinfo"
+)
+
+// Retention reports how long each store keeps data, as Go duration strings such as "720h0m0s".
+type Retention struct {
+	Hot        string `json:"hot" example:"720h0m0s"`
+	FiveMinute string `json:"fiveminute"`
+	Long       string `json:"long"`
+}
+
+// Meta is what the UI footer and range picker need to know about this server.
+type Meta struct {
+	Version   string    `json:"version" doc:"Release version stamped at build time; dev for an unstamped build."`
+	Build     string    `json:"build" doc:"Commit the binary was built from."`
+	Retention Retention `json:"retention"`
+	Now       time.Time `json:"now" doc:"Server time, UTC."`
+}
+
+func (s *server) registerMeta(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "get-meta", Method: http.MethodGet, Path: "/api/v1/meta", Tags: []string{"meta"},
+		Summary: "Version, build, retention and server time",
+	}, func(context.Context, *struct{}) (*struct{ Body Meta }, error) {
+		v, b := buildinfo.Info()
+		return &struct{ Body Meta }{Meta{
+			Version: v, Build: b, Now: s.Clock.Now().UTC(),
+			Retention: Retention{
+				Hot: s.HotRetention.String(), FiveMinute: s.FiveMinuteRetention.String(), Long: s.LongRetention.String(),
+			},
+		}}, nil
+	})
+}

internal/api/ops.go

@@ -2,14 +2,11 @@ package api
 
 import (
 	"context"
-	"encoding/json"
 	"net/http"
-	"time"
 
 	"github.com/danielgtaylor/huma/v2"
 
 	"github.com/v10networks/nmon/internal/discovery"
-	"github.com/v10networks/nmon/internal/poller"
 	"github.com/v10networks/nmon/internal/store"
 )
 
@@ -37,33 +34,6 @@ type DiscoverView struct {
 	Warnings          []string    `json:"warnings"`
 }
 
-// PollerDevice is the poller's view of one device. It uses the slug, never the numeric ID.
-type PollerDevice struct {
-	Device              string     `json:"device" doc:"Device slug."`
-	State               string     `json:"state" doc:"ok or degraded."`
-	Profile             string     `json:"profile"`
-	Interfaces          int        `json:"interfaces" doc:"Interfaces polled."`
-	VarbindsPerRequest  int        `json:"varbinds_per_request"`
-	ConsecutiveFailures int        `json:"consecutive_failures"`
-	MissedDeadlines     int        `json:"missed_deadlines"`
-	IntervalSeconds     float64    `json:"interval_seconds" doc:"Current cadence of the hot group, backoff included."`
-	BaseIntervalSeconds float64    `json:"base_interval_seconds"`
-	LastCycleSeconds    float64    `json:"last_cycle_duration_seconds"`
-	LastCycle           *time.Time `json:"last_cycle,omitempty"`
-	NextCycle           *time.Time `json:"next_cycle,omitempty"`
-	LastError           string     `json:"last_error,omitempty"`
-}
-
-// IdentityReviewView is an open identity review.
-type IdentityReviewView struct {
-	ID         int64           `json:"id"`
-	Device     string          `json:"device,omitempty" doc:"Device slug."`
-	Kind       string          `json:"kind" doc:"ambiguous_match, hardware_replaced or reattach_candidate."`
-	Candidates json.RawMessage `json:"candidates"`
-	Evidence   json.RawMessage `json:"evidence"`
-	CreatedAt  time.Time       `json:"created_at"`
-}
-
 func (s *server) registerOps(api huma.API) {
 	huma.Register(api, huma.Operation{
 		OperationID: "discover-device", Method: "POST", Path: "/api/v1/devices/{slug}/discover", Tags: []string{"devices"},
@@ -72,14 +42,6 @@ func (s *server) registerOps(api huma.API) {
 			"returns what changed. Admin-only once authentication arrives (M6).",
 		Errors: []int{http.StatusNotFound, http.StatusConflict, http.StatusBadGateway},
 	}, s.discoverDevice)
-	huma.Register(api, huma.Operation{
-		OperationID: "poller-health", Method: "GET", Path: "/api/v1/poller/health", Tags: []string{"poller"},
-		Summary: "Poller state of every polled device",
-	}, s.pollerHealth)
-	huma.Register(api, huma.Operation{
-		OperationID: "list-identity-reviews", Method: "GET", Path: "/api/v1/identity-reviews", Tags: []string{"devices"},
-		Summary: "Open identity reviews",
-	}, s.identityReviews)
 }
 
 func (s *server) discoverDevice(ctx context.Context, in *struct {
@@ -121,59 +83,3 @@ func DiscoverViewOf(r *discovery.Report) DiscoverView {
 	}
 	return v
 }
-
-func (s *server) pollerHealth(context.Context, *struct{}) (*struct{ Body []PollerDevice }, error) {
-	out := []PollerDevice{}
-	if s.PollerStatus != nil {
-		for _, d := range s.PollerStatus() {
-			out = append(out, pollerDevice(d))
-		}
-	}
-	return &struct{ Body []PollerDevice }{out}, nil
-}
-
-func pollerDevice(d poller.DeviceStatus) PollerDevice {
-	return PollerDevice{
-		Device: d.Device, State: d.State, Profile: d.Profile, Interfaces: d.Interfaces, VarbindsPerRequest: d.Varbinds,
-		ConsecutiveFailures: d.ConsecutiveFailures, MissedDeadlines: d.MissedDeadlines,
-		IntervalSeconds: d.Interval.Seconds(), BaseIntervalSeconds: d.BaseInterval.Seconds(),
-		LastCycleSeconds: d.LastCycleDuration.Seconds(), LastCycle: tp(d.LastCycle), NextCycle: tp(d.NextCycle),
-		LastError: d.LastError,
-	}
-}
-
-func tp(t time.Time) *time.Time {
-	if t.IsZero() {
-		return nil
-	}
-	u := t.UTC()
-	return &u
-}
-
-func (s *server) identityReviews(ctx context.Context, _ *struct{}) (*struct{ Body []IdentityReviewView }, error) {
-	rows, _, err := s.Store.ListOpenIdentityReviews(ctx, store.Page{Limit: store.MaxLimit})
-	if err != nil {
-		return nil, s.storeErr(err)
-	}
-	out := []IdentityReviewView{}
-	for _, r := range rows {
-		v := IdentityReviewView{
-			ID: r.ID, Kind: r.Kind, Candidates: rawJSON(r.Candidates), Evidence: rawJSON(r.Evidence), CreatedAt: r.CreatedAt.UTC(),
-		}
-		if r.DeviceID != nil {
-			if d, err := s.Store.GetDeviceByID(ctx, *r.DeviceID); err == nil {
-				v.Device = d.Slug
-			}
-		}
-		out = append(out, v)
-	}
-	return &struct{ Body []IdentityReviewView }{out}, nil
-}
-
-func rawJSON(s string) json.RawMessage {
-	if json.Valid([]byte(s)) {
-		return json.RawMessage(s)
-	}
-	b, _ := json.Marshal(s)
-	return b
-}

internal/api/ops_test.go

@@ -4,12 +4,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"strings"
 	"testing"
 
 	"github.com/v10networks/nmon/internal/api"
 	"github.com/v10networks/nmon/internal/fixturegen"
-	"github.com/v10networks/nmon/internal/poller"
 	"github.com/v10networks/nmon/internal/simagent"
 	"github.com/v10networks/nmon/internal/testutil"
 )
@@ -43,24 +41,3 @@ func TestDiscoverEndpointReportsAndRefreshes(t *testing.T) {
 	}
 	isProblem(t, e.do("POST", "/api/v1/devices/no-such-device/discover", ""), 404)
 }
-
-func TestPollerHealthUsesSlugs(t *testing.T) {
-	e := newEnv(t, 0)
-	rec := e.do("GET", "/api/v1/poller/health", "")
-	if rec.Code != 200 || strings.TrimSpace(rec.Body.String()) != "[]" {
-		t.Fatalf("empty health: %d %s", rec.Code, rec.Body)
-	}
-	e.status = []poller.DeviceStatus{{Device: "core-1", DeviceID: 77, State: "degraded", Varbinds: 20, Profile: "juniper-junos"}}
-	rec = e.do("GET", "/api/v1/poller/health", "")
-	if rec.Code != 200 || !strings.Contains(rec.Body.String(), `"device":"core-1"`) || strings.Contains(rec.Body.String(), "77") {
-		t.Fatalf("health: %s", rec.Body)
-	}
-}
-
-func TestIdentityReviewsList(t *testing.T) {
-	e := newEnv(t, 0)
-	rec := e.do("GET", "/api/v1/identity-reviews", "")
-	if rec.Code != 200 || strings.TrimSpace(rec.Body.String()) != "[]" {
-		t.Fatalf("reviews: %d %s", rec.Code, rec.Body)
-	}
-}

internal/api/ops_views_test.go

@@ -0,0 +1,165 @@
+package api_test
+
+import (
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/poller"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+func TestPollerHealthShapeAndTotals(t *testing.T) {
+	e := newEnv(t, 0)
+	d := e.device(t, "Core 1", "", "")
+	var h api.PollerHealth
+	e.getJSON(t, "/api/v1/poller/health", &h)
+	if h.Devices == nil || len(h.Devices) != 0 || h.Totals.WriteQueueCapacity != 2_000_000 {
+		t.Fatalf("empty health: %+v", h)
+	}
+	next := e.clk.Now().Add(30 * time.Second)
+	e.status = []poller.DeviceStatus{
+		{Device: d.Slug, DeviceID: 77, State: store.PollOK, Varbinds: 40, Interval: time.Minute, BaseInterval: time.Minute,
+			LastCycleDuration: 1500 * time.Millisecond, NextCycle: next, Profile: "generic"},
+		{Device: "ghost", State: store.PollDegraded, Varbinds: 20, Interval: 4 * time.Minute, BaseInterval: time.Minute,
+			ConsecutiveFailures: 3, MissedDeadlines: 3, LastError: "snmp: timeout"},
+		{Device: "slow", State: store.PollOK, Interval: 2 * time.Minute, BaseInterval: time.Minute},
+	}
+	rec := e.getJSON(t, "/api/v1/poller/health", &h)
+	assertNoLeaks(t, "poller health", rec.Body.Bytes())
+	if strings.Contains(rec.Body.String(), "77") {
+		t.Errorf("numeric device ID leaked: %s", rec.Body)
+	}
+	if len(h.Devices) != 3 {
+		t.Fatalf("devices: %+v", h.Devices)
+	}
+	c := h.Devices[0]
+	if c.Slug != d.Slug || c.Name != "Core 1" || c.State != "ok" || c.Varbinds != 40 || c.IntervalS != 60 || c.BaseIntervalS != 60 ||
+		c.LastCycleS != 1.5 || c.NextCycle == nil || !c.NextCycle.Equal(next) {
+		t.Errorf("device row: %+v", c)
+	}
+	if g := h.Devices[1]; g.State != "degraded" || g.ConsecutiveFailures != 3 || g.MissedDeadlines != 3 || g.LastError != "snmp: timeout" || g.NextCycle != nil {
+		t.Errorf("degraded row: %+v", g)
+	}
+	if h.Devices[2].State != "backoff" || h.Totals.DevicesDegraded != 2 {
+		t.Errorf("backoff: %+v degraded=%d", h.Devices[2], h.Totals.DevicesDegraded)
+	}
+}
+
+func TestPollerHealthPerMinuteRates(t *testing.T) {
+	e := newEnv(t, 0)
+	pdus := e.reg.Counter("nmon_pdus_total", "path", "poll", "type", "get")
+	e.reg.Counter("nmon_pdus_total", "path", "discovery", "type", "get").Add(99999) // not the poll path
+	timeouts := e.reg.Counter("nmon_snmp_timeouts_total", "path", "poll")
+	missA := e.reg.Counter("nmon_poll_deadline_missed_total", "device", "a")
+	missB := e.reg.Counter("nmon_poll_deadline_missed_total", "device", "b")
+	e.reg.Gauge("nmon_write_queue_depth", "queue", "hot").Set(1234)
+	e.reg.Gauge("nmon_write_queue_depth", "queue", "long").Set(5)
+
+	pdus.Add(600)
+	timeouts.Add(6)
+	missA.Add(3)
+	e.clk.Advance(30 * time.Second)
+	var h api.PollerHealth
+	e.getJSON(t, "/api/v1/poller/health", &h)
+	tot := h.Totals
+	if tot.PDUsPerMin != 1200 || tot.TimeoutsPerMin != 12 || tot.DeadlineMissesPerMin != 6 || tot.WriteQueueDepth != 1234 {
+		t.Errorf("after 30 s: %+v", tot)
+	}
+	pdus.Add(600)
+	timeouts.Add(1)
+	missB.Add(1)
+	e.clk.Advance(30 * time.Second)
+	e.getJSON(t, "/api/v1/poller/health", &h)
+	if tot = h.Totals; tot.PDUsPerMin != 1200 || tot.TimeoutsPerMin != 7 || tot.DeadlineMissesPerMin != 4 {
+		t.Errorf("after 60 s: %+v", tot)
+	}
+	e.clk.Advance(time.Minute) // the window slides: only the last 90 s count
+	e.getJSON(t, "/api/v1/poller/health", &h)
+	if h.Totals.PDUsPerMin != 400 {
+		t.Errorf("sliding window: %+v", h.Totals)
+	}
+}
+
+func TestMeta(t *testing.T) {
+	e := newEnvDeps(t, 0, func(d *api.Deps) {
+		d.HotRetention, d.FiveMinuteRetention, d.LongRetention = 720*time.Hour, 2880*time.Hour, 17520*time.Hour
+	})
+	var m api.Meta
+	rec := e.getJSON(t, "/api/v1/meta", &m)
+	if m.Retention.Hot != "720h0m0s" || m.Retention.FiveMinute != "2880h0m0s" || m.Retention.Long != "17520h0m0s" {
+		t.Errorf("retention: %+v", m.Retention)
+	}
+	if !m.Now.Equal(e.clk.Now()) || m.Version == "" || m.Build == "" || !strings.Contains(rec.Body.String(), `"now":"2026-10-22T12:00:00Z"`) {
+		t.Errorf("meta: %+v %s", m, rec.Body)
+	}
+	for _, s := range []string{m.Retention.Hot, m.Retention.FiveMinute, m.Retention.Long} {
+		if _, err := time.ParseDuration(s); err != nil {
+			t.Errorf("retention %q is not a Go duration", s)
+		}
+	}
+	var def api.Meta
+	newEnv(t, 0).getJSON(t, "/api/v1/meta", &def)
+	if def.Retention.Hot != "720h0m0s" || def.Retention.FiveMinute != "2880h0m0s" || def.Retention.Long != "17520h0m0s" {
+		t.Errorf("default retention (30 d, 120 d, 2 y): %+v", def.Retention)
+	}
+}
+
+func TestQueryAdmissionPerClientAndGlobal(t *testing.T) {
+	var running, peak atomic.Int32
+	release := make(chan struct{})
+	started := make(chan struct{}, 32)
+	e := newEnvDeps(t, 0, func(d *api.Deps) { d.MaxClientQueries, d.MaxGlobalQueries, d.AdmissionWait = 4, 6, 50*time.Millisecond })
+	e.q.fn = func(string, string, time.Time, time.Time, time.Duration) ([]byte, error) {
+		n := running.Add(1)
+		for {
+			p := peak.Load()
+			if n <= p || peak.CompareAndSwap(p, n) {
+				break
+			}
+		}
+		started <- struct{}{}
+		<-release
+		running.Add(-1)
+		return seriesBody(1), nil
+	}
+	var wg sync.WaitGroup
+	do := func(addr string) {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if rec := e.doFrom("GET", "/api/v1/query?q=up", addr); rec.Code != 200 {
+				t.Errorf("admitted query got %d", rec.Code)
+			}
+		}()
+	}
+	for range 4 {
+		do("192.0.2.1:1000")
+	}
+	for range 4 {
+		<-started
+	}
+	// the fifth query from the same client is refused at once; another client is admitted
+	rec := e.doFrom("GET", "/api/v1/query?q=up", "192.0.2.1:2000")
+	isProblem(t, rec, http.StatusTooManyRequests)
+	do("192.0.2.2:1000")
+	do("192.0.2.3:1000")
+	for range 2 {
+		<-started
+	}
+	// six are running: the global limit; a seventh from a fresh client waits, then is refused
+	isProblem(t, e.doFrom("GET", "/api/v1/query?q=up", "192.0.2.4:1000"), http.StatusTooManyRequests)
+	close(release)
+	wg.Wait()
+	if peak.Load() > 6 {
+		t.Errorf("%d queries ran at once, the global cap is 6", peak.Load())
+	}
+	// slots are returned: the client can query again
+	if rec := e.doFrom("GET", "/api/v1/query?q=up", "192.0.2.1:3000"); rec.Code != 200 {
+		t.Errorf("after release: %d %s", rec.Code, rec.Body)
+	}
+}

internal/api/paging.go

@@ -0,0 +1,21 @@
+package api
+
+// PageParams are the paging query parameters every list takes. The maximum is
+// enforced by validation, so a larger per_page is a 422 problem.
+type PageParams struct {
+	Page    int64 `query:"page" minimum:"1" default:"1" doc:"1-based page number."`
+	PerPage int64 `query:"per_page" minimum:"1" maximum:"1000" default:"100" doc:"Items per page; at most 1,000."`
+}
+
+// offset and limit translate the page into store terms.
+func (p PageParams) offset() int64 { return (max(p.Page, 1) - 1) * p.limit() }
+
+func (p PageParams) limit() int64 {
+	if p.PerPage <= 0 {
+		return 100
+	}
+	return p.PerPage
+}
+
+// pageNo is the page number echoed in the response envelope.
+func (p PageParams) pageNo() int64 { return max(p.Page, 1) }

internal/api/parents.go

@@ -0,0 +1,78 @@
+package api
+
+import (
+	"context"
+	"strings"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// parentCycle reports the path a -> b -> ... -> a that making parents the
+// parents of dev would close, or nil when the graph stays acyclic. An edge
+// x -> y means x depends on y. M4's dependency package owns suppression logic;
+// this is only the guard the API needs so a cycle never reaches the store.
+func (s *server) parentCycle(ctx context.Context, dev *store.Device, parents []*store.Device) ([]string, error) {
+	visited := map[int64]bool{}
+	var walk func(cur *store.Device, path []string) ([]string, error)
+	walk = func(cur *store.Device, path []string) ([]string, error) {
+		path = append(path, cur.Slug)
+		if cur.ID == dev.ID {
+			return path, nil
+		}
+		if visited[cur.ID] {
+			return nil, nil
+		}
+		visited[cur.ID] = true
+		links, err := s.Store.ListParents(ctx, cur.ID, "")
+		if err != nil {
+			return nil, err
+		}
+		for _, l := range links {
+			up, err := s.Store.GetDeviceByID(ctx, l.ParentID)
+			if err != nil {
+				return nil, err
+			}
+			if cyc, err := walk(up, path); cyc != nil || err != nil {
+				return cyc, err
+			}
+		}
+		return nil, nil
+	}
+	for _, p := range parents {
+		if cyc, err := walk(p, []string{dev.Slug}); cyc != nil || err != nil {
+			return cyc, err
+		}
+	}
+	return nil, nil
+}
+
+// resolveParents maps parent slugs to unretired devices, rejecting unknown
+// slugs, duplicates and any cycle with a 422 that names the path.
+func (s *server) resolveParents(ctx context.Context, dev *store.Device, slugs []string) ([]*store.Device, error) {
+	seen := map[string]bool{}
+	var out []*store.Device
+	for _, slug := range slugs {
+		if seen[slug] {
+			continue
+		}
+		seen[slug] = true
+		p, err := s.Store.GetDeviceBySlug(ctx, slug)
+		if err != nil || p.PurgedAt != nil || p.RetiredAt != nil {
+			if err != nil && !isNotFound(err) {
+				return nil, s.storeErr(err)
+			}
+			return nil, huma.Error422UnprocessableEntity("parents: no active device with slug " + quote(slug))
+		}
+		out = append(out, p)
+	}
+	cyc, err := s.parentCycle(ctx, dev, out)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	if cyc != nil {
+		return nil, huma.Error422UnprocessableEntity("parents would create a dependency cycle: " + strings.Join(cyc, " -> "))
+	}
+	return out, nil
+}

internal/api/problem.go

@@ -0,0 +1,85 @@
+package api
+
+import (
+	"net/http"
+	"reflect"
+	"sync"
+
+	"github.com/danielgtaylor/huma/v2"
+)
+
+// ProblemDetail is one entry of a problem's errors list (a failed validation, say).
+type ProblemDetail struct {
+	Message  string `json:"message" doc:"Error message text."`
+	Location string `json:"location,omitempty" doc:"Where the error occurred, e.g. query.per_page or body.parents."`
+	Value    any    `json:"value,omitempty" doc:"The offending value."`
+}
+
+// Problem is the RFC 9457 problem details body of every error response. It
+// replaces huma's ErrorModel only to make title and status required, which
+// the generated TypeScript types rely on.
+type Problem struct {
+	Type     string          `json:"type,omitempty" doc:"URI reference identifying the problem type; about:blank when absent."`
+	Title    string          `json:"title" example:"Not Found"`
+	Status   int             `json:"status" example:"404" doc:"HTTP status code."`
+	Detail   string          `json:"detail,omitempty" doc:"Explanation specific to this occurrence."`
+	Instance string          `json:"instance,omitempty"`
+	Errors   []ProblemDetail `json:"errors,omitempty" doc:"Individual errors, e.g. one per invalid parameter."`
+}
+
+// Error returns the detail, falling back to the title.
+func (p *Problem) Error() string {
+	if p.Detail != "" {
+		return p.Detail
+	}
+	return p.Title
+}
+
+// GetStatus is the HTTP status of the response.
+func (p *Problem) GetStatus() int { return p.Status }
+
+// ContentType makes huma answer application/problem+json.
+func (p *Problem) ContentType(ct string) string {
+	if ct == "application/json" {
+		return "application/problem+json"
+	}
+	return ct
+}
+
+// newProblem is huma.NewError for this API.
+func newProblem(status int, msg string, errs ...error) huma.StatusError {
+	p := &Problem{Title: http.StatusText(status), Status: status, Detail: msg}
+	for _, e := range errs {
+		if d, ok := e.(huma.ErrorDetailer); ok {
+			x := d.ErrorDetail()
+			p.Errors = append(p.Errors, ProblemDetail{Message: x.Message, Location: x.Location, Value: x.Value})
+			continue
+		}
+		p.Errors = append(p.Errors, ProblemDetail{Message: e.Error()})
+	}
+	return p
+}
+
+var humaOnce sync.Once
+
+// configureHuma sets huma's package-level switches once: arrays are never
+// nullable, and errors are Problems.
+func configureHuma() {
+	humaOnce.Do(func() {
+		huma.DefaultArrayNullable = false
+		huma.NewError = newProblem
+	})
+}
+
+// addDefaultProblem documents the RFC 9457 default response on every
+// operation, the way the UI contract does, whatever status codes it also lists.
+func addDefaultProblem(oapi *huma.OpenAPI, op *huma.Operation) {
+	if _, ok := op.Responses["default"]; ok {
+		return
+	}
+	schema := oapi.Components.Schemas.Schema(reflect.TypeFor[Problem](), true, "")
+	op.Responses["default"] = &huma.Response{
+		Description: "RFC 9457 problem details",
+		Content:     map[string]*huma.MediaType{"application/problem+json": {Schema: schema}},
+	}
+}

internal/api/query.go

@@ -11,8 +11,6 @@ import (
 	"time"
 
 	"github.com/danielgtaylor/huma/v2"
-
-	"github.com/v10networks/nmon/internal/vmsup"
 )
 
 var jsonObject = map[string]*huma.Response{
@@ -55,6 +53,11 @@ func (s *server) query(ctx context.Context, in *struct {
 			return nil, huma.Error422UnprocessableEntity("time: " + err.Error())
 		}
 	}
+	release, err := s.admit(ctx, true)
+	if err != nil {
+		return nil, err
+	}
+	defer release()
 	qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
 	defer cancel()
 	body, err := s.Querier.Query(qctx, "hot", in.Q, ts)
@@ -88,6 +91,11 @@ func (s *server) queryRange(ctx context.Context, in *struct {
 	if !from.Before(to) {
 		return nil, huma.Error422UnprocessableEntity("from must be before to")
 	}
+	release, err := s.admit(ctx, true)
+	if err != nil {
+		return nil, err
+	}
+	defer release()
 	qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
 	defer cancel()
 	body, err := s.Querier.QueryRange(qctx, "hot", in.Q, from, to, step)
@@ -96,17 +104,8 @@ func (s *server) queryRange(ctx context.Context, in *struct {
 
 // finish applies the series cap and maps store errors to problems.
 func (s *server) finish(ctx context.Context, body []byte, err error) (*huma.StreamResponse, error) {
-	switch {
-	case err == nil:
-	case errors.Is(err, vmsup.ErrStoreDown):
-		return nil, huma.Error503ServiceUnavailable("the metric store is not available")
-	case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded):
-		return nil, huma.Error504GatewayTimeout("query exceeded " + QueryTimeout.String())
-	case strings.Contains(err.Error(), "400 Bad Request") || strings.Contains(err.Error(), "422 Unprocessable"):
-		return nil, huma.Error422UnprocessableEntity(err.Error()) // the store rejected the query
-	default:
-		s.Logger.Warn("query failed", "err", err)
-		return nil, huma.Error502BadGateway("metric store error")
+	if err != nil {
+		return nil, s.vmErr(ctx, err)
 	}
 	if n := countSeries(body); n > MaxSeries {
 		return nil, huma.Error422UnprocessableEntity(
@@ -115,6 +114,23 @@ func (s *server) finish(ctx context.Context, body []byte, err error) (*huma.Stre
 	return raw(body), nil
 }
 
+func errStoreDown() error {
+	return huma.Error503ServiceUnavailable("the metric store is not available")
+}
+
+func errQueryTimeout() error {
+	return huma.Error504GatewayTimeout("query exceeded " + QueryTimeout.String())
+}
+
+func errStoreFailed() error { return huma.Error502BadGateway("metric store error") }
+
+// isBadQuery reports a rejection by the store of the query itself.
+func isBadQuery(err error) bool {
+	return strings.Contains(err.Error(), "400 Bad Request") || strings.Contains(err.Error(), "422 Unprocessable")
+}
+
+func badQuery(err error) error { return huma.Error422UnprocessableEntity(err.Error()) }
+
 // countSeries returns len(data.result) of a Prometheus-style response.
 func countSeries(body []byte) int {
 	var r struct {

internal/api/rates.go

@@ -0,0 +1,125 @@
+package api
+
+import (
+	"context"
+	"strings"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// liveWindow is the lookback of the "live" rates on device and search pages.
+const liveWindow = "5m"
+
+// ifRates holds the five-minute rates of one interface by the counter they came
+// from; pick chooses between the 64-bit and 32-bit counters.
+type ifRates struct {
+	hcIn, hcOut, in32, out32, inErr, outErr *float64
+}
+
+// rateKinds maps the k label of the batched query to the metric and whether it is a bits/s rate.
+var rateKinds = []struct {
+	k, metric string
+	bits      bool
+}{
+	{"in_hc", "ifHCInOctets", true}, {"out_hc", "ifHCOutOctets", true},
+	{"in_32", "ifInOctets", true}, {"out_32", "ifOutOctets", true},
+	{"in_err", "ifInErrors", false}, {"out_err", "ifOutErrors", false},
+}
+
+// ratesExpr is one MetricsQL expression yielding every live rate of the interfaces
+// matched by match (label matchers), each series tagged with k. When uptime is set
+// it also yields the device's last sysUpTime under k="uptime".
+func ratesExpr(match string, uptime bool) string {
+	parts := make([]string, 0, len(rateKinds)+1)
+	for _, k := range rateKinds {
+		scale := ""
+		if k.bits {
+			scale = "*8"
+		}
+		parts = append(parts, `label_set(rate(`+k.metric+`{`+match+`}[`+liveWindow+`])`+scale+`, "k", `+quote(k.k)+`)`)
+	}
+	if uptime {
+		parts = append(parts, `label_set(last_over_time(sysUpTime{`+match+`}[10m]), "k", "uptime")`)
+	}
+	return strings.Join(parts, " or ")
+}
+
+// liveRates runs the batched query. A store that is down, busy or slow yields
+// no rates rather than failing the page: the fields are null, as for unknown.
+func (s *server) liveRates(ctx context.Context, match string, uptime bool) (map[string]*ifRates, *float64) {
+	release, err := s.admit(ctx, false)
+	if err != nil {
+		return nil, nil
+	}
+	defer release()
+	qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
+	defer cancel()
+	body, err := s.Querier.Query(qctx, "hot", ratesExpr(match, uptime), s.Clock.Now())
+	if err != nil {
+		s.Logger.Warn("live rates unavailable", "err", err)
+		return nil, nil
+	}
+	rows, err := parseVM(body)
+	if err != nil {
+		s.Logger.Warn("live rates unreadable", "err", err)
+		return nil, nil
+	}
+	return indexRates(rows)
+}
+
+// indexRates sorts result rows into per-interface rates and the device uptime.
+func indexRates(rows []vmSeries) (map[string]*ifRates, *float64) {
+	out := map[string]*ifRates{}
+	var uptime *float64
+	for _, r := range rows {
+		if r.Value == nil || !r.Value.OK {
+			continue
+		}
+		v := r.Value.V
+		if r.Metric["k"] == "uptime" {
+			uptime = &v
+			continue
+		}
+		id := r.Metric["iface"]
+		x := out[id]
+		if x == nil {
+			x = &ifRates{}
+			out[id] = x
+		}
+		switch r.Metric["k"] {
+		case "in_hc":
+			x.hcIn = &v
+		case "out_hc":
+			x.hcOut = &v
+		case "in_32":
+			x.in32 = &v
+		case "out_32":
+			x.out32 = &v
+		case "in_err":
+			x.inErr = &v
+		case "out_err":
+			x.outErr = &v
+		}
+	}
+	return out, uptime
+}
+
+// bps returns the in and out rates from the counters the interface really has;
+// an unreliable interface has none.
+func (x *ifRates) bps(i *store.Interface) (in, out *float64) {
+	if x == nil || i.Unreliable {
+		return nil, nil
+	}
+	if i.HasHCCounters {
+		return x.hcIn, x.hcOut
+	}
+	return x.in32, x.out32
+}
+
+// errs returns the error rates, none for an unreliable interface.
+func (x *ifRates) errs(i *store.Interface) (in, out *float64) {
+	if x == nil || i.Unreliable {
+		return nil, nil
+	}
+	return x.inErr, x.outErr
+}

internal/api/reach.go

@@ -0,0 +1,141 @@
+package api
+
+import (
+	"context"
+	"time"
+
+	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// Reachability values shown in lists and on device pages.
+const (
+	ReachUp          = "up"
+	ReachDown        = "down"
+	ReachWarning     = "warning"
+	ReachUnknown     = "unknown"
+	ReachMaintenance = "maintenance"
+)
+
+// ReachabilitySource decides each device's reachability. The result is keyed by
+// device slug and holds one of the Reach* values. M4 replaces the default with
+// the alerter's state (dependencies, maintenance windows, suppression).
+type ReachabilitySource interface {
+	Reachability(ctx context.Context, devs []store.Device) map[string]string
+}
+
+// persistedReachability derives reachability from what is stored right now:
+// the ICMP and SNMP up series in the hot store, else the device's last_seen
+// age against the poll interval. It knows nothing of dependencies.
+type persistedReachability struct {
+	q        Querier
+	clk      clock.Clock
+	interval time.Duration
+}
+
+// upLookback is how stale an up series may be and still count.
+const upLookback = "5m"
+
+// maxScoped is the most devices a query names explicitly; beyond it all devices are fetched.
+const maxScoped = 200
+
+func (p *persistedReachability) Reachability(ctx context.Context, devs []store.Device) map[string]string {
+	icmp, snmp := p.upSeries(ctx, devs)
+	now := p.clk.Now()
+	out := make(map[string]string, len(devs))
+	for i := range devs {
+		d := &devs[i]
+		out[d.Slug] = reachabilityOf(d, icmp[d.Slug], snmp[d.Slug], now, p.interval)
+	}
+	return out
+}
+
+// upSeries returns the last nmon_icmp_up and nmon_snmp_up per device slug. A
+// store that cannot be queried yields empty maps, and the caller falls back to last_seen.
+func (p *persistedReachability) upSeries(ctx context.Context, devs []store.Device) (icmp, snmp map[string]*float64) {
+	match := ""
+	if len(devs) > 0 && len(devs) <= maxScoped {
+		slugs := make([]string, len(devs))
+		for i, d := range devs {
+			slugs[i] = d.Slug
+		}
+		match = `device=~` + quote(regexAlt(slugs))
+	}
+	get := func(metric string) map[string]*float64 {
+		body, err := p.q.Query(ctx, "hot", `last_over_time(`+metric+`{`+match+`}[`+upLookback+`])`, p.clk.Now())
+		if err != nil {
+			return nil
+		}
+		rows, err := parseVM(body)
+		if err != nil {
+			return nil
+		}
+		m := make(map[string]*float64, len(rows))
+		for _, r := range rows {
+			if r.Value != nil && r.Value.OK {
+				v := r.Value.V
+				m[r.Metric["device"]] = &v
+			}
+		}
+		return m
+	}
+	return get("nmon_icmp_up"), get("nmon_snmp_up")
+}
+
+// reachabilityOf is the decision for one device. icmp and snmp are the last up
+// values (1 or 0) or nil when the store has none.
+func reachabilityOf(d *store.Device, icmp, snmp *float64, now time.Time, interval time.Duration) string {
+	switch {
+	case d.RetiredAt != nil || !d.Enabled:
+		return ReachUnknown
+	case d.PollState == store.PollMaintenance:
+		return ReachMaintenance
+	}
+	var probes []bool
+	for _, v := range []*float64{icmp, snmp} {
+		if v != nil {
+			probes = append(probes, *v >= 1)
+		}
+	}
+	if len(probes) == 0 {
+		return staleness(d, now, interval)
+	}
+	up := 0
+	for _, ok := range probes {
+		if ok {
+			up++
+		}
+	}
+	switch {
+	case up == 0:
+		return ReachDown
+	case up < len(probes), d.PollState == store.PollDegraded, d.PollState == store.PollBackoff:
+		return ReachWarning
+	}
+	return ReachUp
+}
+
+// staleness judges a device with no up series by when it last answered: not
+// heard from in three poll intervals is down.
+func staleness(d *store.Device, now time.Time, interval time.Duration) string {
+	if d.LastSeen == nil {
+		return ReachUnknown
+	}
+	if now.Sub(*d.LastSeen) > 3*interval {
+		return ReachDown
+	}
+	if d.PollState == store.PollDegraded || d.PollState == store.PollBackoff {
+		return ReachWarning
+	}
+	return ReachUp
+}
+
+// pollStateOf maps the stored state to the API enum (ok, degraded, backoff);
+// maintenance is a reachability, not a poll state.
+func pollStateOf(d *store.Device) string {
+	switch d.PollState {
+	case store.PollDegraded, store.PollBackoff:
+		return d.PollState
+	}
+	return store.PollOK
+}

internal/api/reviews.go

@@ -0,0 +1,232 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"slices"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// ReviewCandidate is an interface an identity review is undecided about.
+type ReviewCandidate struct {
+	IfaceID string   `json:"iface_id" doc:"Interface ULID."`
+	IfName  string   `json:"ifname,omitempty"`
+	Score   *float64 `json:"score,omitempty"`
+}
+
+// IdentityReview is an open decision about which interfaces (or which hardware) are the same.
+type IdentityReview struct {
+	ID         string            `json:"id" doc:"Opaque review identifier."`
+	Kind       string            `json:"kind" enum:"ambiguous_match,hardware_replaced,reattach_candidate"`
+	Device     DeviceRef         `json:"device"`
+	Candidates []ReviewCandidate `json:"candidates"`
+	Evidence   []string          `json:"evidence" doc:"What the matcher saw, as key: value lines."`
+	CreatedAt  time.Time         `json:"created_at"`
+}
+
+// IdentityReviewList is a page of IdentityReview.
+type IdentityReviewList struct {
+	Items   []IdentityReview `json:"items"`
+	Page    int64            `json:"page"`
+	PerPage int64            `json:"per_page"`
+	Total   int64            `json:"total"`
+}
+
+// ResolveBody is the body of POST /identity-reviews/{id}/resolve.
+type ResolveBody struct {
+	Action string `json:"action" enum:"merge,keep_both" doc:"merge: keep one interface and retire the others. keep_both: close the review and leave every row as it is."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
+	Keep   string `json:"keep,omitempty" doc:"With merge: the interface ULID to keep; it must be one of the review's candidates."`
+}
+
+func (s *server) registerReviews(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "list-identity-reviews", Method: http.MethodGet, Path: "/api/v1/identity-reviews", Tags: []string{"devices"},
+		Summary: "Open identity reviews, oldest first",
+	}, s.listReviews)
+	huma.Register(api, huma.Operation{
+		OperationID: "resolve-identity-review", Method: http.MethodPost, Path: "/api/v1/identity-reviews/{id}/resolve",
+		Tags: []string{"devices"}, Summary: "Resolve an identity review",
+		Description: "merge keeps the chosen interface ID and retires the other rows of the review (their history stays in the " +
+			"metric store under their own IDs). keep_both closes the review without changing any row. Both are audited.",
+		Errors: []int{http.StatusNotFound, http.StatusConflict, http.StatusUnprocessableEntity},
+	}, s.resolveReview)
+}
+
+func (s *server) listReviews(ctx context.Context, in *struct{ PageParams }) (*struct{ Body IdentityReviewList }, error) {
+	rows, total, err := s.Store.ListOpenIdentityReviews(ctx, store.Page{Offset: in.offset(), Limit: in.limit()})
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	out := IdentityReviewList{Items: make([]IdentityReview, 0, len(rows)), Page: in.pageNo(), PerPage: in.limit(), Total: total}
+	for i := range rows {
+		v, err := s.reviewView(ctx, &rows[i])
+		if err != nil {
+			return nil, s.storeErr(err)
+		}
+		out.Items = append(out.Items, v)
+	}
+	return &struct{ Body IdentityReviewList }{out}, nil
+}
+
+func (s *server) reviewView(ctx context.Context, r *store.IdentityReview) (IdentityReview, error) {
+	v := IdentityReview{
+		ID: strconv.FormatInt(r.ID, 10), Kind: r.Kind, CreatedAt: r.CreatedAt.UTC(),
+		Candidates: reviewCandidates(r), Evidence: evidenceLines(r.Evidence),
+	}
+	if r.DeviceID != nil {
+		d, err := s.Store.GetDeviceByID(ctx, *r.DeviceID)
+		if err != nil {
+			return v, err
+		}
+		v.Device = DeviceRef{Slug: d.Slug, Name: d.Name}
+	}
+	return v, nil
+}
+
+// reviewCandidates reads the interface candidates of an ambiguous_match review.
+// Other kinds decide about hardware or retired devices, not interfaces.
+func reviewCandidates(r *store.IdentityReview) []ReviewCandidate {
+	out := []ReviewCandidate{}
+	if r.Kind != store.ReviewAmbiguousMatch {
+		return out
+	}
+	var cs []struct {
+		ID     string `json:"id"`
+		IfName string `json:"ifname"`
+	}
+	if json.Unmarshal([]byte(r.Candidates), &cs) != nil {
+		return out
+	}
+	for _, c := range cs {
+		out = append(out, ReviewCandidate{IfaceID: c.ID, IfName: c.IfName})
+	}
+	return out
+}
+
+// evidenceLines renders the evidence object as sorted "key: value" lines.
+func evidenceLines(raw string) []string {
+	out := []string{}
+	var m map[string]json.RawMessage
+	if json.Unmarshal([]byte(raw), &m) != nil {
+		if raw != "" {
+			out = append(out, raw)
+		}
+		return out
+	}
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		var str string
+		if json.Unmarshal(m[k], &str) == nil {
+			out = append(out, k+": "+str)
+			continue
+		}
+		out = append(out, k+": "+string(m[k]))
+	}
+	return out
+}
+
+// reviewIfaceIDs lists every interface an ambiguous review concerns: the
+// frozen candidates and the new rows the split gave the found ports.
+func reviewIfaceIDs(r *store.IdentityReview) []string {
+	var ids []string
+	for _, c := range reviewCandidates(r) {
+		ids = append(ids, c.IfaceID)
+	}
+	var ev struct {
+		Found []struct {
+			NewID string `json:"new_id"`
+		} `json:"found"`
+	}
+	if json.Unmarshal([]byte(r.Evidence), &ev) == nil {
+		for _, f := range ev.Found {
+			if f.NewID != "" && !slices.Contains(ids, f.NewID) {
+				ids = append(ids, f.NewID)
+			}
+		}
+	}
+	return ids
+}
+
+func (s *server) resolveReview(ctx context.Context, in *struct {
+	ID   string `path:"id" doc:"Review identifier."`
+	Body ResolveBody
+}) (*struct{ Body IdentityReview }, error) {
+	id, err := strconv.ParseInt(in.ID, 10, 64)
+	if err != nil {
+		return nil, huma.Error404NotFound("not found")
+	}
+	r, err := s.Store.GetIdentityReview(ctx, id)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	if r.ResolvedAt != nil {
+		return nil, huma.Error409Conflict("review already resolved")
+	}
+	losers, err := mergeLosers(r, in.Body)
+	if err != nil {
+		return nil, err
+	}
+	if err := s.applyResolution(ctx, r, in.Body, losers); err != nil {
+		return nil, s.storeErr(err)
+	}
+	v, err := s.reviewView(ctx, r)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	return &struct{ Body IdentityReview }{v}, nil
+}
+
+// mergeLosers validates the resolution and returns the interface IDs a merge retires.
+func mergeLosers(r *store.IdentityReview, b ResolveBody) ([]string, error) {
+	if b.Action == "keep_both" {
+		return nil, nil
+	}
+	if r.Kind != store.ReviewAmbiguousMatch {
+		return nil, huma.Error422UnprocessableEntity("merge applies to ambiguous_match reviews; use keep_both to close a " + r.Kind + " review")
+	}
+	if b.Keep == "" {
+		return nil, huma.Error422UnprocessableEntity("merge needs keep: the interface ID to keep")
+	}
+	ids := reviewIfaceIDs(r)
+	if !slices.Contains(ids, b.Keep) {
+		return nil, huma.Error422UnprocessableEntity(fmt.Sprintf("keep %s is not one of this review's interfaces", b.Keep))
+	}
+	return slices.DeleteFunc(slices.Clone(ids), func(x string) bool { return x == b.Keep }), nil
+}
+
+// applyResolution retires the losers, closes the review and audits both in one transaction.
+func (s *server) applyResolution(ctx context.Context, r *store.IdentityReview, b ResolveBody, losers []string) error {
+	return s.Store.WithTx(ctx, func(tx store.Store) error {
+		var retired []string
+		for _, id := range losers {
+			i, err := tx.GetInterface(ctx, id)
+			if err != nil {
+				return err
+			}
+			if i.RetiredAt != nil {
+				continue
+			}
+			if err := tx.RetireInterface(ctx, id); err != nil {
+				return err
+			}
+			retired = append(retired, id)
+		}
+		if err := tx.ResolveIdentityReview(ctx, r.ID, "anonymous"); err != nil {
+			return err
+		}
+		return tx.AppendAudit(ctx, "anonymous", "identity_review."+b.Action, "identity_review:"+strconv.FormatInt(r.ID, 10),
+			map[string]any{"kind": r.Kind, "interfaces": reviewIfaceIDs(r)}, map[string]any{"kept": b.Keep, "retired": retired})
+	})
+}

internal/api/reviews_test.go

@@ -0,0 +1,140 @@
+package api_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// ambiguous opens a review shaped like discovery's: two frozen old rows are the
+// candidates and the split gave the found port a third, new row.
+func (e *env) ambiguous(t *testing.T) (d *store.Device, old1, old2, fresh store.Interface, review *store.IdentityReview) {
+	t.Helper()
+	d = e.device(t, "Stack", "yyz", "core")
+	old1 = e.iface(t, d, 1, "Gi1/0/1", "")
+	old2 = e.iface(t, d, 2, "Gi2/0/1", "")
+	fresh = e.iface(t, d, 3, "Gi1/0/1", "")
+	cands := []map[string]any{
+		{"id": old1.ID, "ifindex": 1, "ifname": "Gi1/0/1", "retired": false},
+		{"id": old2.ID, "ifindex": 2, "ifname": "Gi2/0/1", "retired": false},
+	}
+	ev := map[string]any{"key": old1.ID + "," + old2.ID, "reasons": []string{"multiple_candidates"},
+		"found": []map[string]any{{"ifindex": 3, "new_id": fresh.ID}}}
+	r, err := e.st.CreateIdentityReview(t.Context(), &d.ID, store.ReviewAmbiguousMatch, cands, ev)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d, old1, old2, fresh, r
+}
+
+func TestIdentityReviewListShape(t *testing.T) {
+	e := newEnv(t, 0)
+	d, old1, old2, _, r := e.ambiguous(t)
+	hw, err := e.st.CreateIdentityReview(t.Context(), &d.ID, store.ReviewHardwareReplaced,
+		[]map[string]string{{"serial": "A"}, {"serial": "B"}}, map[string]any{"old_serial": "A", "new_serial": "B"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var l api.IdentityReviewList
+	rec := e.getJSON(t, "/api/v1/identity-reviews", &l)
+	assertNoLeaks(t, "identity reviews", rec.Body.Bytes())
+	if l.Total != 2 || len(l.Items) != 2 || l.Page != 1 || l.PerPage != 100 {
+		t.Fatalf("list: %+v", l)
+	}
+	a := l.Items[0]
+	if a.ID != fmt.Sprint(r.ID) || a.Kind != "ambiguous_match" || a.Device.Slug != d.Slug || a.Device.Name != "Stack" ||
+		len(a.Candidates) != 2 || a.Candidates[0].IfaceID != old1.ID || a.Candidates[0].IfName != "Gi1/0/1" || a.Candidates[1].IfaceID != old2.ID {
+		t.Errorf("ambiguous review: %+v", a)
+	}
+	joined := strings.Join(a.Evidence, "|")
+	if !strings.Contains(joined, "reasons: ") || !strings.Contains(joined, "multiple_candidates") {
+		t.Errorf("evidence: %v", a.Evidence)
+	}
+	if b := l.Items[1]; b.ID != fmt.Sprint(hw.ID) || b.Kind != "hardware_replaced" || b.Candidates == nil || len(b.Candidates) != 0 ||
+		!strings.Contains(strings.Join(b.Evidence, "|"), "new_serial: B") {
+		t.Errorf("hardware review: %+v", b)
+	}
+	e.getJSON(t, "/api/v1/identity-reviews?per_page=1&page=2", &l)
+	if len(l.Items) != 1 || l.Total != 2 || l.Items[0].Kind != "hardware_replaced" {
+		t.Errorf("page 2: %+v", l)
+	}
+}
+
+func TestResolveMergeRetiresTheLosersAndKeepsTheChosenID(t *testing.T) {
+	e := newEnv(t, 0)
+	d, old1, old2, fresh, r := e.ambiguous(t)
+	path := fmt.Sprintf("/api/v1/identity-reviews/%d/resolve", r.ID)
+	for body, status := range map[string]int{
+		`{"action":"merge"}`: 422, // keep missing
+		`{"action":"merge","keep":"01ARZ3NDEKTSV4RRFFQ69G5FAV"}`: 422, // not in the review
+		`{"action":"explode"}`: 422,
+		`{}`:                   422,
+	} {
+		isProblem(t, e.do("POST", path, body), status)
+	}
+	if open, _, _ := e.st.ListOpenIdentityReviews(t.Context(), store.Page{}); len(open) != 1 {
+		t.Fatal("a rejected resolve must leave the review open")
+	}
+	rec := e.do("POST", path, `{"action":"merge","keep":"`+old1.ID+`"}`)
+	if rec.Code != 200 {
+		t.Fatalf("merge: %d %s", rec.Code, rec.Body)
+	}
+	var v api.IdentityReview
+	_ = json.Unmarshal(rec.Body.Bytes(), &v)
+	if v.ID != fmt.Sprint(r.ID) || v.Kind != "ambiguous_match" {
+		t.Errorf("response: %+v", v)
+	}
+	ifs, _ := e.st.ListInterfaces(t.Context(), d.ID, true)
+	retired := map[string]bool{}
+	for _, i := range ifs {
+		retired[i.ID] = i.RetiredAt != nil
+	}
+	if retired[old1.ID] || !retired[old2.ID] || !retired[fresh.ID] || len(ifs) != 3 {
+		t.Errorf("retired after merge: %v", retired)
+	}
+	if open, _, _ := e.st.ListOpenIdentityReviews(t.Context(), store.Page{}); len(open) != 0 {
+		t.Errorf("review still open: %+v", open)
+	}
+	audit, _, _ := e.st.ListAudit(t.Context(), store.Page{})
+	if len(audit) != 1 || audit[0].Action != "identity_review.merge" || !strings.Contains(audit[0].After, old2.ID) ||
+		!strings.Contains(audit[0].After, old1.ID) || !strings.Contains(audit[0].Object, fmt.Sprint(r.ID)) {
+		t.Errorf("audit: %+v", audit)
+	}
+	isProblem(t, e.do("POST", path, `{"action":"keep_both"}`), 409) // already resolved
+}
+
+func TestResolveKeepBothChangesNoRowsAndIsAudited(t *testing.T) {
+	e := newEnv(t, 0)
+	d, _, _, _, r := e.ambiguous(t)
+	rec := e.do("POST", fmt.Sprintf("/api/v1/identity-reviews/%d/resolve", r.ID), `{"action":"keep_both"}`)
+	if rec.Code != 200 {
+		t.Fatalf("keep_both: %d %s", rec.Code, rec.Body)
+	}
+	ifs, _ := e.st.ListInterfaces(t.Context(), d.ID, true)
+	for _, i := range ifs {
+		if i.RetiredAt != nil {
+			t.Errorf("%s retired by keep_both", i.ID)
+		}
+	}
+	audit, _, _ := e.st.ListAudit(t.Context(), store.Page{})
+	if len(audit) != 1 || audit[0].Action != "identity_review.keep_both" {
+		t.Errorf("audit: %+v", audit)
+	}
+}
+
+func TestResolveErrorsAndNonInterfaceReviews(t *testing.T) {
+	e := newEnv(t, 0)
+	d := e.device(t, "Box", "", "")
+	hw, _ := e.st.CreateIdentityReview(t.Context(), &d.ID, store.ReviewHardwareReplaced, []string{"x"}, map[string]string{"a": "b"})
+	isProblem(t, e.do("POST", "/api/v1/identity-reviews/999/resolve", `{"action":"keep_both"}`), 404)
+	isProblem(t, e.do("POST", "/api/v1/identity-reviews/abc/resolve", `{"action":"keep_both"}`), 404)
+	path := fmt.Sprintf("/api/v1/identity-reviews/%d/resolve", hw.ID)
+	isProblem(t, e.do("POST", path, `{"action":"merge","keep":"x"}`), 422)
+	if rec := e.do("POST", path, `{"action":"keep_both"}`); rec.Code != 200 {
+		t.Errorf("closing a hardware_replaced review: %d %s", rec.Code, rec.Body)
+	}
+}

internal/api/vmjson.go

@@ -0,0 +1,106 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+)
+
+// vmPoint is one sample of a Prometheus-style response. OK is false for NaN and
+// infinities, which JSON cannot carry and a graph must show as a gap.
+type vmPoint struct {
+	TS int64 // unix seconds
+	V  float64
+	OK bool
+}
+
+func (p *vmPoint) UnmarshalJSON(b []byte) error {
+	var raw [2]json.RawMessage
+	if err := json.Unmarshal(b, &raw); err != nil {
+		return err
+	}
+	var ts float64
+	if err := json.Unmarshal(raw[0], &ts); err != nil {
+		return err
+	}
+	var str string
+	if err := json.Unmarshal(raw[1], &str); err != nil {
+		return err
+	}
+	v, err := strconv.ParseFloat(str, 64)
+	p.TS = int64(math.Round(ts))
+	p.V, p.OK = v, err == nil && !math.IsNaN(v) && !math.IsInf(v, 0)
+	return nil
+}
+
+// vmSeries is one result row: its labels and either one sample (instant query)
+// or many (range query).
+type vmSeries struct {
+	Metric map[string]string `json:"metric"`
+	Value  *vmPoint          `json:"value"`
+	Values []vmPoint         `json:"values"`
+}
+
+var errVMStatus = errors.New("metric store reported an error")
+
+// parseVM decodes the result rows of an instant or range response.
+func parseVM(body []byte) ([]vmSeries, error) {
+	var r struct {
+		Status string `json:"status"`
+		Error  string `json:"error"`
+		Data   struct {
+			Result []vmSeries `json:"result"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(body, &r); err != nil {
+		return nil, fmt.Errorf("decode metric store response: %w", err)
+	}
+	if r.Status != "success" {
+		return nil, fmt.Errorf("%w: %s", errVMStatus, r.Error)
+	}
+	return r.Data.Result, nil
+}
+
+// quote renders s as a PromQL string literal.
+func quote(s string) string {
+	var b strings.Builder
+	b.WriteByte('"')
+	for _, r := range s {
+		switch r {
+		case '"', '\\':
+			b.WriteByte('\\')
+			b.WriteRune(r)
+		case '\n':
+			b.WriteString(`\n`)
+		default:
+			b.WriteRune(r)
+		}
+	}
+	b.WriteByte('"')
+	return b.String()
+}
+
+// regexAlt joins literal values into an anchored-by-VM regex alternation.
+func regexAlt(vals []string) string {
+	esc := make([]string, len(vals))
+	for i, v := range vals {
+		esc[i] = quoteMeta(v)
+	}
+	return strings.Join(esc, "|")
+}
+
+// quoteMeta escapes regex metacharacters; slugs and ULIDs rarely need it, names might.
+func quoteMeta(s string) string {
+	const meta = `\.+*?()|[]{}^$`
+	var b strings.Builder
+	for _, r := range s {
+		if strings.ContainsRune(meta, r) {
+			b.WriteByte('\\')
+		}
+		b.WriteRune(r)
+	}
+	return b.String()
+}

internal/app/integration_test.go

@@ -229,7 +229,7 @@ func TestServeDiscoverEndpointHealthAndTraps(t *testing.T) {
 		}
 		body, _ = io.ReadAll(resp.Body)
 		resp.Body.Close()
-		if strings.Contains(string(body), `"device":"`+dev.Slug+`"`) && strings.Contains(string(body), `"profile":"juniper-junos"`) {
+		if strings.Contains(string(body), `"slug":"`+dev.Slug+`"`) && strings.Contains(string(body), `"profile":"juniper-junos"`) {
 			break
 		}
 		if time.Now().After(deadline) {

internal/app/openapi.go

@@ -0,0 +1,7 @@
+package app
+
+import "github.com/v10networks/nmon/internal/api"
+
+// OpenAPISpec returns the generated OpenAPI 3.1 document for `nmon openapi`.
+// It lives here because only internal/app may import internal/api.
+func OpenAPISpec() ([]byte, error) { return api.Spec() }

internal/app/wiring.go

@@ -18,6 +18,7 @@ import (
 	"github.com/v10networks/nmon/internal/store"
 	"github.com/v10networks/nmon/internal/traps"
 	"github.com/v10networks/nmon/internal/vmsup"
+	"github.com/v10networks/nmon/internal/webui"
 )
 
 // Each start* method brings one subsystem up and registers, with onStop, how
@@ -178,14 +179,19 @@ func (a *App) logTrapsDown(addr string, err error) {
 func (a *App) startAPI(s *settings) error {
 	handler := api.New(api.Deps{
 		Store: a.st, Querier: vmsup.NewQuerier(a.sup), Poller: a.poller, SNMPPort: s.SNMPPort, Logger: a.log,
-		Profiles: a.profiles, PollerStatus: a.poller.Status, AfterDiscover: a.afterDiscover,
+		Profiles: a.profiles, PollerStatus: a.poller.Status, AfterDiscover: a.afterDiscover, PollInterval: s.PollInterval,
+		HotRetention: s.cfg.Retention.Hot.D(), FiveMinuteRetention: s.cfg.Retention.FiveMinute.D(), LongRetention: s.cfg.Retention.Long.D(),
 	})
+	ui, err := webui.Handler(webui.Dist(), handler)
+	if err != nil {
+		return err
+	}
 	ln, err := net.Listen("tcp", s.Listen)
 	if err != nil {
 		return fmt.Errorf("listen: %w", err)
 	}
 	a.ln = ln
-	a.srv = &http.Server{Handler: handler, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
+	a.srv = &http.Server{Handler: ui, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
 	go func() {
 		if err := a.srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
 			a.serveErr <- err

internal/buildinfo/buildinfo.go

@@ -0,0 +1,32 @@
+// Package buildinfo carries the version stamped into the binary. The release
+// build sets Version and Build with
+// -ldflags "-X github.com/v10networks/nmon/internal/buildinfo.Version=v1.2.3 -X ...Build=<commit>".
+package buildinfo
+
+import "runtime/debug"
+
+// Version and Build are set by -ldflags -X; the defaults mark an unstamped build.
+var (
+	Version = "dev"
+	Build   = ""
+)
+
+// Info returns the version and the build identifier. An unstamped build falls
+// back to the VCS revision Go embeds, so a `go build` binary still says which commit it is.
+func Info() (version, build string) {
+	version, build = Version, Build
+	if build != "" {
+		return version, build
+	}
+	if bi, ok := debug.ReadBuildInfo(); ok {
+		for _, s := range bi.Settings {
+			if s.Key == "vcs.revision" {
+				build = s.Value
+			}
+		}
+	}
+	if build == "" {
+		build = "unknown"
+	}
+	return version, build
+}

internal/deps/cycle.go

@@ -0,0 +1,105 @@
+package deps
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// CycleError reports a dependency cycle. Path lists the slugs following
+// child-to-parent links and ends where it starts: a, c, b, a means a's parent
+// is c, c's parent is b and b's parent is a.
+type CycleError struct{ Path []string }
+
+func (e *CycleError) Error() string {
+	return "dependency cycle: " + strings.Join(e.Path, " -> ")
+}
+
+// Validate returns a *CycleError when edges (child to parent) contain a cycle,
+// including a device that is its own parent. Callers validate the full
+// resulting edge set before writing anything, so a batch is all-or-nothing.
+func Validate(edges []Edge) error {
+	parents := map[string][]string{}
+	for _, e := range edges {
+		parents[e.Child] = append(parents[e.Child], e.Parent)
+	}
+	nodes := make([]string, 0, len(parents))
+	for c, ps := range parents {
+		sort.Strings(ps)
+		nodes = append(nodes, c)
+	}
+	sort.Strings(nodes)
+	const (
+		visiting = 1
+		done     = 2
+	)
+	mark := map[string]int{}
+	var stack []string
+	var walk func(n string) []string
+	walk = func(n string) []string {
+		mark[n] = visiting
+		stack = append(stack, n)
+		for _, p := range parents[n] {
+			switch mark[p] {
+			case visiting:
+				return cyclePath(stack, p)
+			case 0:
+				if c := walk(p); c != nil {
+					return c
+				}
+			}
+		}
+		stack = stack[:len(stack)-1]
+		mark[n] = done
+		return nil
+	}
+	for _, n := range nodes {
+		if mark[n] == 0 {
+			if p := walk(n); p != nil {
+				return &CycleError{Path: p}
+			}
+		}
+	}
+	return nil
+}
+
+// cyclePath cuts the DFS stack at the first occurrence of start and closes the loop.
+func cyclePath(stack []string, start string) []string {
+	i := len(stack) - 1
+	for stack[i] != start {
+		i--
+	}
+	out := append([]string(nil), stack[i:]...)
+	return append(out, start)
+}
+
+// With returns the graph's edges with each child in changes given exactly the
+// listed parents (an empty list clears them), the shape of a PUT parents or a
+// CSV import batch. Pass the result to Validate.
+func (g *Graph) With(changes map[string][]string) []Edge {
+	var out []Edge
+	for _, e := range g.Edges() {
+		if _, replaced := changes[e.Child]; !replaced {
+			out = append(out, e)
+		}
+	}
+	kids := make([]string, 0, len(changes))
+	for c := range changes {
+		kids = append(kids, c)
+	}
+	sort.Strings(kids)
+	for _, c := range kids {
+		for _, p := range changes[c] {
+			out = append(out, Edge{Child: c, Parent: p})
+		}
+	}
+	return out
+}
+
+// Check validates the graph with changes applied.
+func (g *Graph) Check(changes map[string][]string) error {
+	if err := Validate(g.With(changes)); err != nil {
+		return fmt.Errorf("parents: %w", err)
+	}
+	return nil
+}

internal/deps/deps_test.go

@@ -0,0 +1,138 @@
+package deps
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+const topoPath = "../../testdata/topologies/redundant.yaml"
+
+func down(set ...string) func(string) bool {
+	m := map[string]bool{}
+	for _, s := range set {
+		m[s] = true
+	}
+	return func(d string) bool { return m[d] }
+}
+
+func TestRedundantTopologyLoadsAndSuppressesOnlyWhenEveryParentIsDown(t *testing.T) {
+	topo, err := LoadTopology(topoPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(topo.Devices) != 24 {
+		t.Fatalf("devices = %d, want 24", len(topo.Devices))
+	}
+	g := topo.Graph()
+	if sup, _ := g.Suppressed("dist-1", down("core-1")); sup {
+		t.Fatal("one core down must not suppress a dual-homed switch")
+	}
+	sup, by := g.Suppressed("dist-1", down("core-1", "core-2"))
+	if !sup || !reflect.DeepEqual(by, []string{"core-1", "core-2"}) {
+		t.Fatalf("both cores down: suppressed=%v by=%v", sup, by)
+	}
+	if sup, _ := g.Suppressed("core-1", down("core-2")); sup {
+		t.Fatal("a device without parents is never suppressed")
+	}
+}
+
+func TestShadowedPropagatesThroughLevels(t *testing.T) {
+	g := New()
+	topo, _ := LoadTopology(topoPath)
+	g.Replace(topo.Edges())
+	sh := g.Shadowed(down("core-1", "core-2"))
+	if len(sh) != 22 {
+		t.Fatalf("shadowed = %d, want 22 (2 dist + 20 access)", len(sh))
+	}
+	if got := len(g.ShadowedBelow("core-1", sh)); got != 22 {
+		t.Fatalf("below core-1 = %d, want 22", got)
+	}
+	if sh := g.Shadowed(down("core-1")); len(sh) != 0 {
+		t.Fatalf("one core down shadows %v", sh)
+	}
+	// dist-1 itself down: only its own access switches are shadowed.
+	sh = g.Shadowed(down("dist-1"))
+	if len(sh) != 10 || sh["access-02"] {
+		t.Fatalf("dist-1 down shadows %d (%v)", len(sh), sh["access-02"])
+	}
+}
+
+func TestValidateReturnsTheCyclePath(t *testing.T) {
+	edges := []Edge{{"a", "c"}, {"c", "b"}, {"b", "a"}}
+	var ce *CycleError
+	err := Validate(edges)
+	if !errors.As(err, &ce) {
+		t.Fatalf("err = %v", err)
+	}
+	if want := []string{"a", "c", "b", "a"}; !reflect.DeepEqual(ce.Path, want) {
+		t.Fatalf("path = %v, want %v", ce.Path, want)
+	}
+	if err := Validate([]Edge{{"x", "x"}}); err == nil {
+		t.Fatal("self parent accepted")
+	}
+	if err := Validate([]Edge{{"a", "b"}, {"a", "c"}, {"b", "d"}, {"c", "d"}}); err != nil {
+		t.Fatalf("diamond rejected: %v", err)
+	}
+}
+
+func TestBatchIsAllOrNothing(t *testing.T) {
+	g := New()
+	g.Replace([]Edge{{"b", "a"}, {"c", "b"}})
+	// each change alone is fine; together they close a loop a -> c -> b -> a
+	if err := g.Check(map[string][]string{"a": {"c"}}); err == nil {
+		t.Fatal("cycle through existing edges accepted")
+	}
+	if err := g.Check(map[string][]string{"d": {"c"}, "e": {"d"}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := g.Check(map[string][]string{"d": {"e"}, "e": {"d"}}); err == nil {
+		t.Fatal("cycle inside the batch accepted")
+	}
+	if len(g.Edges()) != 2 {
+		t.Fatal("Check mutated the graph")
+	}
+	if err := g.Check(map[string][]string{"b": nil, "a": {"c"}}); err != nil {
+		t.Fatalf("clearing b's parents should break the loop: %v", err)
+	}
+}
+
+func TestShadowedToleratesStoredCycle(t *testing.T) {
+	g := New()
+	g.Replace([]Edge{{"a", "b"}, {"b", "a"}})
+	if sh := g.Shadowed(down()); len(sh) != 0 {
+		t.Fatalf("shadowed = %v", sh)
+	}
+}
+
+func big(n int) *Graph {
+	var edges []Edge
+	for i := 0; i < n; i++ {
+		edges = append(edges, Edge{fmt.Sprintf("d%04d", i), fmt.Sprintf("d%04d", i/3)})
+		if i > 5 {
+			edges = append(edges, Edge{fmt.Sprintf("d%04d", i), fmt.Sprintf("d%04d", i/3-1)})
+		}
+	}
+	g := New()
+	g.Replace(edges)
+	return g
+}
+
+func BenchmarkSuppressed2000(b *testing.B) {
+	g := big(2000)
+	isDown := down("d0001", "d0002")
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		g.Suppressed("d1999", isDown)
+	}
+}
+
+func BenchmarkShadowedPass2000(b *testing.B) {
+	g := big(2000)
+	isDown := down("d0001", "d0002", "d0003")
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		g.Shadowed(isDown)
+	}
+}

internal/deps/graph.go

@@ -0,0 +1,118 @@
+// Package deps holds the device dependency DAG (device_parents, relation
+// network) and the rules that need only its shape: cycle detection on write
+// and "every parent is down" suppression. It knows nothing about how a device
+// is judged down; callers pass that in.
+package deps
+
+import (
+	"context"
+	"sort"
+	"sync/atomic"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// Edge says Parent is upstream of Child. Both are device slugs.
+type Edge struct{ Child, Parent string }
+
+// Source is the part of the store the graph loads from.
+type Source interface {
+	ListDevices(ctx context.Context, f store.DeviceFilter, p store.Page) ([]store.Device, int64, error)
+	ListAllParents(ctx context.Context, relation string) ([]store.DeviceParent, error)
+}
+
+// snapshot is an immutable view; Graph swaps whole snapshots so readers never lock.
+type snapshot struct {
+	parents  map[string][]string
+	children map[string][]string
+}
+
+// Graph is the network dependency DAG, safe for concurrent use. Reads are
+// lock-free, so the evaluator can ask about 2,000 devices every few seconds.
+type Graph struct{ cur atomic.Pointer[snapshot] }
+
+// New returns an empty graph.
+func New() *Graph {
+	g := &Graph{}
+	g.cur.Store(&snapshot{})
+	return g
+}
+
+// Replace swaps in edges. It does not check for cycles (use Validate on
+// writes); the readers here tolerate one in case the database holds one.
+func (g *Graph) Replace(edges []Edge) {
+	s := &snapshot{parents: map[string][]string{}, children: map[string][]string{}}
+	for _, e := range edges {
+		s.parents[e.Child] = append(s.parents[e.Child], e.Parent)
+		s.children[e.Parent] = append(s.children[e.Parent], e.Child)
+	}
+	for _, m := range []map[string][]string{s.parents, s.children} {
+		for k := range m {
+			sort.Strings(m[k])
+		}
+	}
+	g.cur.Store(s)
+}
+
+// Load rebuilds the graph from the store's relation=network links.
+func (g *Graph) Load(ctx context.Context, src Source) error {
+	var devs []store.Device
+	for off := int64(0); ; off += store.MaxLimit {
+		page, total, err := src.ListDevices(ctx, store.DeviceFilter{}, store.Page{Offset: off, Limit: store.MaxLimit})
+		if err != nil {
+			return err
+		}
+		devs = append(devs, page...)
+		if int64(len(devs)) >= total || len(page) == 0 {
+			break
+		}
+	}
+	links, err := src.ListAllParents(ctx, store.RelNetwork)
+	if err != nil {
+		return err
+	}
+	g.Replace(EdgesFrom(devs, links))
+	return nil
+}
+
+// EdgesFrom maps stored links to slug edges, skipping links to devices not in
+// devs (retired ones) and any relation other than network.
+func EdgesFrom(devs []store.Device, links []store.DeviceParent) []Edge {
+	slug := make(map[int64]string, len(devs))
+	for i := range devs {
+		slug[devs[i].ID] = devs[i].Slug
+	}
+	out := make([]Edge, 0, len(links))
+	for _, l := range links {
+		c, cok := slug[l.DeviceID]
+		p, pok := slug[l.ParentID]
+		if cok && pok && l.Relation == store.RelNetwork {
+			out = append(out, Edge{Child: c, Parent: p})
+		}
+	}
+	return out
+}
+
+// Parents returns the sorted parent slugs of child. The slice is shared; do not modify it.
+func (g *Graph) Parents(child string) []string { return g.cur.Load().parents[child] }
+
+// Children returns the sorted child slugs of parent. The slice is shared; do not modify it.
+func (g *Graph) Children(parent string) []string { return g.cur.Load().children[parent] }
+
+// Edges returns every edge sorted by child then parent.
+func (g *Graph) Edges() []Edge {
+	s := g.cur.Load()
+	out := make([]Edge, 0, len(s.parents))
+	for c, ps := range s.parents {
+		for _, p := range ps {
+			out = append(out, Edge{Child: c, Parent: p})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Child != out[j].Child {
+			return out[i].Child < out[j].Child
+		}
+		return out[i].Parent < out[j].Parent
+	})
+	return out
+}

internal/deps/suppress.go

@@ -0,0 +1,96 @@
+package deps
+
+// Suppressed reports whether child's down alert is suppressed: it has at
+// least one parent and every parent is down. It returns the parents that
+// explain it. isDown is the caller's view of reachability; for a multi-level
+// chain a parent that is itself suppressed-down is down in that view, and
+// Shadowed extends it to devices that are unreachable only because of their parents.
+func (g *Graph) Suppressed(child string, isDown func(device string) bool) (bool, []string) {
+	ps := g.cur.Load().parents[child]
+	if len(ps) == 0 {
+		return false, nil
+	}
+	for _, p := range ps {
+		if !isDown(p) {
+			return false, nil
+		}
+	}
+	return true, ps
+}
+
+// Shadowed returns the devices that are cut off by their parents: every
+// parent is down or itself shadowed, whatever the device's own state. Such a
+// device cannot be polled through the network, so it is part of an outage
+// even before its own probes fail. Cycles in stored data are treated as not shadowed.
+func (g *Graph) Shadowed(isDown func(device string) bool) map[string]bool {
+	s := g.cur.Load()
+	c := &shadowCalc{parents: s.parents, isDown: isDown, mark: make(map[string]int, len(s.parents))}
+	out := map[string]bool{}
+	for n := range s.parents {
+		if c.shadowed(n) {
+			out[n] = true
+		}
+	}
+	return out
+}
+
+// shadowCalc memoises the recursive verdict per device.
+type shadowCalc struct {
+	parents map[string][]string
+	isDown  func(string) bool
+	mark    map[string]int
+}
+
+const (
+	markVisiting = iota + 1
+	markNo
+	markYes
+)
+
+func (c *shadowCalc) shadowed(n string) bool {
+	switch c.mark[n] {
+	case markYes:
+		return true
+	case markNo, markVisiting:
+		return false
+	}
+	ps := c.parents[n]
+	if len(ps) == 0 {
+		c.mark[n] = markNo
+		return false
+	}
+	c.mark[n] = markVisiting
+	all := true
+	for _, p := range ps {
+		if !c.isDown(p) && !c.shadowed(p) {
+			all = false
+			break
+		}
+	}
+	c.mark[n] = markNo
+	if all {
+		c.mark[n] = markYes
+	}
+	return all
+}
+
+// ShadowedBelow returns the shadowed devices reachable from root through
+// shadowed devices only: the ones root's outage accounts for.
+func (g *Graph) ShadowedBelow(root string, shadowed map[string]bool) []string {
+	s := g.cur.Load()
+	seen := map[string]bool{root: true}
+	queue := []string{root}
+	var out []string
+	for len(queue) > 0 {
+		n := queue[0]
+		queue = queue[1:]
+		for _, c := range s.children[n] {
+			if shadowed[c] && !seen[c] {
+				seen[c] = true
+				out = append(out, c)
+				queue = append(queue, c)
+			}
+		}
+	}
+	return out
+}

internal/deps/topology.go

@@ -0,0 +1,85 @@
+package deps
+
+import (
+	"fmt"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Topology is a fixture file describing devices and their parents, used by
+// tests and the M4 gate (testdata/topologies/*.yaml).
+type Topology struct {
+	Name    string       `yaml:"name"`
+	Devices []TopoDevice `yaml:"devices"`
+}
+
+// TopoDevice is one device of a topology; Parents are slugs.
+type TopoDevice struct {
+	Slug    string   `yaml:"slug"`
+	Role    string   `yaml:"role"`
+	Kind    string   `yaml:"kind"`
+	Parents []string `yaml:"parents"`
+}
+
+// ParseTopology decodes and checks a topology: unique slugs, known parents, no cycle.
+func ParseTopology(b []byte) (*Topology, error) {
+	var t Topology
+	if err := yaml.Unmarshal(b, &t); err != nil {
+		return nil, fmt.Errorf("topology: %w", err)
+	}
+	known := map[string]bool{}
+	for _, d := range t.Devices {
+		if d.Slug == "" || known[d.Slug] {
+			return nil, fmt.Errorf("topology %s: empty or duplicate slug %q", t.Name, d.Slug)
+		}
+		known[d.Slug] = true
+	}
+	for _, d := range t.Devices {
+		for _, p := range d.Parents {
+			if !known[p] {
+				return nil, fmt.Errorf("topology %s: %s has unknown parent %q", t.Name, d.Slug, p)
+			}
+		}
+	}
+	if err := Validate(t.Edges()); err != nil {
+		return nil, fmt.Errorf("topology %s: %w", t.Name, err)
+	}
+	return &t, nil
+}
+
+// LoadTopology reads a topology file.
+func LoadTopology(path string) (*Topology, error) {
+	b, err := os.ReadFile(path) //nolint:gosec // path comes from the test or gate caller
+	if err != nil {
+		return nil, fmt.Errorf("topology: %w", err)
+	}
+	return ParseTopology(b)
+}
+
+// Edges lists the child-to-parent edges.
+func (t *Topology) Edges() []Edge {
+	var out []Edge
+	for _, d := range t.Devices {
+		for _, p := range d.Parents {
+			out = append(out, Edge{Child: d.Slug, Parent: p})
+		}
+	}
+	return out
+}
+
+// Graph builds the dependency graph.
+func (t *Topology) Graph() *Graph {
+	g := New()
+	g.Replace(t.Edges())
+	return g
+}
+
+// Slugs lists every device slug in file order.
+func (t *Topology) Slugs() []string {
+	out := make([]string, len(t.Devices))
+	for i, d := range t.Devices {
+		out[i] = d.Slug
+	}
+	return out
+}

internal/openapidiff/cmd/openapidiff/main.go

@@ -0,0 +1,42 @@
+// Command openapidiff compares two OpenAPI documents and exits 1 when the
+// second is not backwards compatible with the first.
+//
+//	openapidiff old.json new.json
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/v10networks/nmon/internal/openapidiff"
+)
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Fprintln(os.Stderr, "usage: openapidiff old.json new.json")
+		os.Exit(2)
+	}
+	oldSpec, err := os.ReadFile(os.Args[1])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "openapidiff:", err)
+		os.Exit(2)
+	}
+	newSpec, err := os.ReadFile(os.Args[2])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "openapidiff:", err)
+		os.Exit(2)
+	}
+	changes, err := openapidiff.Compare(oldSpec, newSpec)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "openapidiff:", err)
+		os.Exit(2)
+	}
+	for _, c := range changes {
+		fmt.Println("INCOMPATIBLE:", c)
+	}
+	if len(changes) > 0 {
+		fmt.Printf("%d incompatible change(s)\n", len(changes))
+		os.Exit(1)
+	}
+	fmt.Println("openapi: compatible")
+}

internal/openapidiff/openapidiff.go

@@ -0,0 +1,194 @@
+// Package openapidiff reports backwards-incompatible changes between two
+// OpenAPI 3 documents, so a released client keeps working against a newer
+// server: removed routes, removed response fields, changed field types, new
+// required request inputs and removed enum values.
+package openapidiff
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// Breaking is one incompatible change.
+type Breaking struct {
+	Where string // e.g. "GET /api/v1/devices response 200 .items[].slug"
+	What  string
+}
+
+func (b Breaking) String() string { return b.Where + ": " + b.What }
+
+type doc = map[string]any
+
+// Compare returns the incompatible changes going from oldSpec to newSpec (JSON
+// documents), sorted for stable output. Additions are never reported.
+func Compare(oldSpec, newSpec []byte) ([]Breaking, error) {
+	var o, n doc
+	if err := json.Unmarshal(oldSpec, &o); err != nil {
+		return nil, fmt.Errorf("old spec: %w", err)
+	}
+	if err := json.Unmarshal(newSpec, &n); err != nil {
+		return nil, fmt.Errorf("new spec: %w", err)
+	}
+	c := &comparer{old: o, new: n, seen: map[string]bool{}}
+	c.paths()
+	sort.Slice(c.out, func(i, j int) bool { return c.out[i].String() < c.out[j].String() })
+	return c.out, nil
+}
+
+type comparer struct {
+	old, new doc
+	out      []Breaking
+	seen     map[string]bool // schema pairs already compared (breaks $ref cycles)
+}
+
+func (c *comparer) add(where, format string, args ...any) {
+	c.out = append(c.out, Breaking{Where: where, What: fmt.Sprintf(format, args...)})
+}
+
+var methods = []string{"get", "put", "post", "delete", "patch", "head", "options"}
+
+func obj(v any) map[string]any {
+	m, _ := v.(map[string]any)
+	return m
+}
+
+func (c *comparer) paths() {
+	op, np := obj(c.old["paths"]), obj(c.new["paths"])
+	for _, p := range sortedKeys(op) {
+		nitem := obj(np[p])
+		if nitem == nil {
+			c.add(p, "route removed")
+			continue
+		}
+		for _, m := range methods {
+			oo := obj(obj(op[p])[m])
+			if oo == nil {
+				continue
+			}
+			no := obj(nitem[m])
+			if no == nil {
+				c.add(strings.ToUpper(m)+" "+p, "method removed")
+				continue
+			}
+			c.operation(strings.ToUpper(m)+" "+p, oo, no)
+		}
+	}
+}
+
+func (c *comparer) operation(where string, o, n map[string]any) {
+	c.parameters(where, o, n)
+	c.requestBody(where, o, n)
+	c.responses(where, o, n)
+}
+
+// paramKey identifies a parameter by location and name.
+func paramKey(p map[string]any) string { return fmt.Sprint(p["in"], ":", p["name"]) }
+
+func (c *comparer) parameters(where string, o, n map[string]any) {
+	olds := map[string]map[string]any{}
+	for _, p := range c.paramList(c.old, o) {
+		olds[paramKey(p)] = p
+	}
+	for _, p := range c.paramList(c.new, n) {
+		k := paramKey(p)
+		op, existed := olds[k]
+		if !existed {
+			if p["required"] == true {
+				c.add(where, "new required parameter %s", k)
+			}
+			continue
+		}
+		if op["required"] != true && p["required"] == true {
+			c.add(where, "parameter %s became required", k)
+		}
+		c.schema(where+" parameter "+k, deref(c.old, obj(op["schema"])), deref(c.new, obj(p["schema"])), request)
+	}
+}
+
+func (c *comparer) paramList(d doc, op map[string]any) []map[string]any {
+	var out []map[string]any
+	list, _ := op["parameters"].([]any)
+	for _, p := range list {
+		if m := deref(d, obj(p)); m != nil {
+			out = append(out, m)
+		}
+	}
+	return out
+}
+
+func (c *comparer) requestBody(where string, o, n map[string]any) {
+	ob, nb := deref(c.old, obj(o["requestBody"])), deref(c.new, obj(n["requestBody"]))
+	if nb == nil {
+		return
+	}
+	if ob == nil {
+		if nb["required"] == true {
+			c.add(where, "request body is now required")
+		}
+		return
+	}
+	if ob["required"] != true && nb["required"] == true {
+		c.add(where, "request body became required")
+	}
+	c.content(where+" request", ob, nb, request)
+}
+
+func (c *comparer) responses(where string, o, n map[string]any) {
+	or, nr := obj(o["responses"]), obj(n["responses"])
+	for _, code := range sortedKeys(or) {
+		if !strings.HasPrefix(code, "2") {
+			continue
+		}
+		nresp := deref(c.new, obj(nr[code]))
+		if nresp == nil {
+			c.add(where, "response %s removed", code)
+			continue
+		}
+		c.content(where+" response "+code, deref(c.old, obj(or[code])), nresp, response)
+	}
+}
+
+// content compares the schema of each media type the old document declared.
+func (c *comparer) content(where string, o, n map[string]any, dir direction) {
+	oc, nc := obj(o["content"]), obj(n["content"])
+	for _, mt := range sortedKeys(oc) {
+		nm := obj(nc[mt])
+		if nm == nil {
+			c.add(where, "media type %s removed", mt)
+			continue
+		}
+		c.schema(where+" "+mt, deref(c.old, obj(obj(oc[mt])["schema"])), deref(c.new, obj(nm["schema"])), dir)
+	}
+}
+
+func sortedKeys(m map[string]any) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// deref follows a local "#/..." $ref chain; unresolvable refs yield nil.
+func deref(d doc, m map[string]any) map[string]any {
+	for i := 0; m != nil && i < 32; i++ {
+		ref, ok := m["$ref"].(string)
+		if !ok {
+			return m
+		}
+		m = lookup(d, ref)
+	}
+	return m
+}
+
+func lookup(d doc, ref string) map[string]any {
+	cur := any(d)
+	for _, part := range strings.Split(strings.TrimPrefix(ref, "#/"), "/") {
+		part = strings.ReplaceAll(strings.ReplaceAll(part, "~1", "/"), "~0", "~")
+		cur = obj(cur)[part]
+	}
+	return obj(cur)
+}

internal/openapidiff/openapidiff_test.go

@@ -0,0 +1,97 @@
+package openapidiff
+
+import (
+	"strings"
+	"testing"
+)
+
+// spec builds a small document: GET /things (query param), POST /things (body), and a shared Thing schema.
+func spec(thing, listParams, postBody string) []byte {
+	return []byte(`{"openapi":"3.1.0","paths":{
+	 "/things":{
+	  "get":{"parameters":[` + listParams + `],"responses":{"200":{"description":"ok","content":{"application/json":{"schema":{
+	    "type":"object","properties":{"items":{"type":"array","items":{"$ref":"#/components/schemas/Thing"}}}}}}}}},
+	  "post":{"requestBody":{"content":{"application/json":{"schema":` + postBody + `}}},"responses":{"201":{"description":"made"}}}},
+	 "/other":{"get":{"responses":{"200":{"description":"ok"}}}}},
+	 "components":{"schemas":{"Thing":` + thing + `}}}`)
+}
+
+const (
+	baseThing = `{"type":"object","required":["slug"],"properties":{"slug":{"type":"string"},"n":{"type":"integer"},"state":{"type":"string","enum":["up","down"]}}}`
+	baseBody  = `{"type":"object","required":["name"],"properties":{"name":{"type":"string"},"site":{"type":"string"}}}`
+	baseParam = `{"name":"q","in":"query","schema":{"type":"string"}}`
+)
+
+func TestCompare(t *testing.T) {
+	tests := []struct {
+		name      string
+		thing     string
+		params    string
+		body      string
+		want      string // substring of the single expected finding; "" means compatible
+		dropOther bool
+	}{
+		{name: "identical", thing: baseThing, params: baseParam, body: baseBody},
+		{name: "added response field and optional param are fine",
+			thing:  strings.Replace(baseThing, `"n":`, `"extra":{"type":"string"},"n":`, 1),
+			params: baseParam + `,{"name":"page","in":"query","schema":{"type":"integer"}}`, body: baseBody},
+		{name: "removed response field", thing: strings.Replace(baseThing, `"n":{"type":"integer"},`, "", 1),
+			params: baseParam, body: baseBody, want: `response field "n" removed`},
+		{name: "changed field type", thing: strings.Replace(baseThing, `"integer"`, `"string"`, 1),
+			params: baseParam, body: baseBody, want: `type changed from "integer" to "string"`},
+		{name: "removed enum value", thing: strings.Replace(baseThing, `["up","down"]`, `["up"]`, 1),
+			params: baseParam, body: baseBody, want: "enum value down removed"},
+		{name: "added enum value is fine", thing: strings.Replace(baseThing, `["up","down"]`, `["up","down","warn"]`, 1),
+			params: baseParam, body: baseBody},
+		{name: "new required parameter", thing: baseThing,
+			params: baseParam + `,{"name":"site","in":"query","required":true,"schema":{"type":"string"}}`, body: baseBody,
+			want: "new required parameter query:site"},
+		{name: "new required request field", thing: baseThing, params: baseParam,
+			body: strings.Replace(baseBody, `["name"]`, `["name","site"]`, 1), want: `request field "site" became required`},
+		{name: "brand new required request field", thing: baseThing, params: baseParam,
+			body: strings.Replace(strings.Replace(baseBody, `["name"]`, `["name","zone"]`, 1), `"site"`, `"zone"`, 1),
+			want: `new required request field "zone"`},
+		{name: "removed route", thing: baseThing, params: baseParam, body: baseBody, dropOther: true, want: "/other: route removed"},
+		{name: "nullable to non-null is still the same type",
+			thing: strings.Replace(baseThing, `"integer"`, `["integer","null"]`, 1), params: baseParam, body: baseBody},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			oldSpec := spec(baseThing, baseParam, baseBody)
+			newSpec := spec(tc.thing, tc.params, tc.body)
+			if tc.dropOther {
+				newSpec = []byte(strings.Replace(string(newSpec), `"/other"`, `"/renamed"`, 1))
+			}
+			got, err := Compare(oldSpec, newSpec)
+			if err != nil {
+				t.Fatal(err)
+			}
+			switch {
+			case tc.want == "" && len(got) != 0:
+				t.Fatalf("unexpected findings: %v", got)
+			case tc.want != "" && (len(got) != 1 || !strings.Contains(got[0].String(), tc.want)):
+				t.Fatalf("want one finding containing %q, got %v", tc.want, got)
+			}
+		})
+	}
+}
+
+func TestRemovedMethodAndRecursiveSchemas(t *testing.T) {
+	oldSpec := `{"paths":{"/a":{"get":{"responses":{"200":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/Node"}}}}}},
+	 "delete":{"responses":{"204":{}}}}},
+	 "components":{"schemas":{"Node":{"type":"object","properties":{"child":{"$ref":"#/components/schemas/Node"},"id":{"type":"string"}}}}}}`
+	newSpec := strings.Replace(oldSpec, `"delete":{"responses":{"204":{}}}`, `"put":{"responses":{"200":{}}}`, 1)
+	got, err := Compare([]byte(oldSpec), []byte(newSpec))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got[0].String() != "DELETE /a: method removed" {
+		t.Fatalf("got %v", got)
+	}
+}
+
+func TestCompareRejectsBadJSON(t *testing.T) {
+	if _, err := Compare([]byte("{"), []byte("{}")); err == nil {
+		t.Fatal("want error")
+	}
+}

internal/openapidiff/schema.go

@@ -0,0 +1,155 @@
+package openapidiff
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+)
+
+// direction says whether a schema describes what clients send or what they receive.
+type direction int
+
+const (
+	request direction = iota
+	response
+)
+
+// schema compares two resolved schemas. Request schemas must not gain required
+// members; response schemas must not lose members, change type or lose enum values.
+func (c *comparer) schema(where string, o, n map[string]any, dir direction) {
+	if o == nil || n == nil {
+		return
+	}
+	key := fmt.Sprintf("%p|%p|%d", o, n, dir)
+	if c.seen[key] {
+		return
+	}
+	c.seen[key] = true
+	o, n = c.flatten(c.old, o), c.flatten(c.new, n)
+	c.types(where, o, n)
+	c.enums(where, o, n)
+	c.properties(where, o, n, dir)
+	c.schema(where+"[]", deref(c.old, obj(o["items"])), deref(c.new, obj(n["items"])), dir)
+}
+
+// flatten merges allOf members into one schema so properties are comparable.
+func (c *comparer) flatten(d doc, s map[string]any) map[string]any {
+	parts, _ := s["allOf"].([]any)
+	if len(parts) == 0 {
+		return s
+	}
+	out := map[string]any{}
+	props := map[string]any{}
+	var req []any
+	for _, p := range append([]any{s}, parts...) {
+		m := c.flatten(d, deref(d, obj(p)))
+		for k, v := range m {
+			if k == "properties" {
+				for pk, pv := range obj(v) {
+					props[pk] = pv
+				}
+			} else if k == "required" {
+				l, _ := v.([]any)
+				req = append(req, l...)
+			} else if k != "allOf" {
+				out[k] = v
+			}
+		}
+	}
+	out["properties"], out["required"] = props, req
+	return out
+}
+
+// typeSet is the non-null JSON types a schema accepts, sorted.
+func typeSet(s map[string]any) string {
+	var ts []string
+	switch t := s["type"].(type) {
+	case string:
+		ts = []string{t}
+	case []any:
+		for _, x := range t {
+			ts = append(ts, fmt.Sprint(x))
+		}
+	}
+	out := ts[:0]
+	for _, t := range ts {
+		if t != "null" {
+			out = append(out, t)
+		}
+	}
+	sort.Strings(out)
+	return strings.Join(out, "|")
+}
+
+func (c *comparer) types(where string, o, n map[string]any) {
+	ot, nt := typeSet(o), typeSet(n)
+	if ot != "" && ot != nt {
+		c.add(where, "type changed from %q to %q", ot, nt)
+	}
+}
+
+func (c *comparer) enums(where string, o, n map[string]any) {
+	oe, _ := o["enum"].([]any)
+	ne, _ := n["enum"].([]any)
+	if len(oe) == 0 {
+		return
+	}
+	if len(ne) == 0 {
+		return // enum constraint dropped: more permissive
+	}
+	for _, v := range oe {
+		if !containsValue(ne, v) {
+			c.add(where, "enum value %v removed", v)
+		}
+	}
+}
+
+func containsValue(list []any, v any) bool {
+	for _, x := range list {
+		if reflect.DeepEqual(x, v) {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *comparer) properties(where string, o, n map[string]any, dir direction) {
+	op, np := obj(o["properties"]), obj(n["properties"])
+	if dir == response {
+		for _, name := range sortedKeys(op) {
+			if _, ok := np[name]; !ok {
+				c.add(where, "response field %q removed", name)
+			}
+		}
+	} else {
+		c.newRequired(where, o, n)
+	}
+	for _, name := range sortedKeys(op) {
+		if nv, ok := np[name]; ok {
+			c.schema(where+"."+name, deref(c.old, obj(op[name])), deref(c.new, obj(nv)), dir)
+		}
+	}
+}
+
+// newRequired reports request members that are required now but were not before.
+func (c *comparer) newRequired(where string, o, n map[string]any) {
+	was := map[string]bool{}
+	list, _ := o["required"].([]any)
+	for _, r := range list {
+		was[fmt.Sprint(r)] = true
+	}
+	op := obj(o["properties"])
+	list, _ = n["required"].([]any)
+	for _, r := range list {
+		name := fmt.Sprint(r)
+		if name == "$schema" || was[name] {
+			continue
+		}
+		if _, existed := op[name]; existed {
+			c.add(where, "request field %q became required", name)
+		} else {
+			c.add(where, "new required request field %q", name)
+		}
+	}
+}

internal/reach/config.go

@@ -0,0 +1,75 @@
+package reach
+
+import (
+	"log/slog"
+	"time"
+
+	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/selfmon"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// Policy thresholds and timing.
+const (
+	DefaultInterval   = 5 * time.Second
+	DefaultRefresh    = 30 * time.Second
+	DefaultICMPDown   = 45 * time.Second
+	DefaultSNMPFails  = 2
+	DefaultNoData     = 5 * time.Minute
+	DefaultPushAbsent = 5 * time.Minute
+	DefaultCheckFails = 3
+	DefaultWarnHold   = 5 * time.Minute
+	FlapWindow        = 10 * time.Minute
+	FlapMax           = 5
+	FlapHold          = 15 * time.Minute
+)
+
+// Options configures an Evaluator. Store, Notifier and Clock are required.
+type Options struct {
+	Store    Store
+	Notifier Notifier
+	Clock    clock.Clock
+	Log      *slog.Logger
+	Registry *selfmon.Registry
+
+	// ICMPBlocked resolves a device's profile flag; nil means never blocked.
+	ICMPBlocked func(d *store.Device) bool
+	// InMaintenance reports whether a maintenance window covers the device at
+	// that instant; nil means no windows. The poller's maintenance state also counts.
+	InMaintenance func(slug string, at time.Time) bool
+
+	Interval, Refresh time.Duration // evaluation pass and device/graph reload periods
+	ICMPDown          time.Duration // continuous ICMP failure that counts as down
+	SNMPFails         int           // consecutive SNMP failures that count as failed
+	NoData            time.Duration
+	PushAbsent        time.Duration
+	// WarnHold is how long an icmp_filtered or snmp_unreachable warning must
+	// last before it is announced, so a real outage that fails the second
+	// signal a few seconds later sends only device_down.
+	WarnHold time.Duration
+}
+
+func (o *Options) defaults() {
+	if o.Log == nil {
+		o.Log = slog.Default()
+	}
+	if o.Registry == nil {
+		o.Registry = selfmon.Default
+	}
+	set := func(p *time.Duration, v time.Duration) {
+		if *p <= 0 {
+			*p = v
+		}
+	}
+	set(&o.Interval, DefaultInterval)
+	set(&o.Refresh, DefaultRefresh)
+	set(&o.ICMPDown, DefaultICMPDown)
+	set(&o.NoData, DefaultNoData)
+	set(&o.PushAbsent, DefaultPushAbsent)
+	if o.SNMPFails <= 0 {
+		o.SNMPFails = DefaultSNMPFails
+	}
+	if o.WarnHold == 0 {
+		o.WarnHold = DefaultWarnHold
+	}
+}

internal/reach/deliver.go

@@ -0,0 +1,110 @@
+package reach
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// deliver sends the row's pending notification if it is still valid and not
+// held back by suppression. A firing notice for a suppressed device is held
+// and sent only if the device is still down once its parents recover; a
+// resolution is sent only if the firing was.
+func (e *Evaluator) deliver(ctx context.Context, now time.Time, p *pass, r *store.Alert) {
+	sup, by := e.g.Suppressed(r.DeviceSlug, p.eff)
+	reason := ""
+	if sup && isActive(r.State) {
+		reason = joinSlugs(by)
+	}
+	if reason != r.SuppressedBy {
+		r.SuppressedBy = reason
+		e.save(ctx, r)
+	}
+	n := r.NotifyPending
+	if n == "" {
+		return
+	}
+	if !pendingValid(r, n) {
+		r.NotifyPending = ""
+		e.save(ctx, r)
+		return
+	}
+	if sup && n != NotifyResolved {
+		return
+	}
+	tr, group := e.transition(now, p, r, n)
+	if group == "" || !p.sent[group] {
+		if err := e.o.Notifier.Notify(ctx, tr); err != nil {
+			e.o.Registry.Counter("nmon_reach_notify_errors_total", "rule", r.Rule).Inc()
+			e.log.Warn("reachability notify failed, will retry", "key", r.SeriesKey, "state", n, "err", err)
+			return
+		}
+	}
+	if group != "" {
+		p.sent[group] = true
+	}
+	r.NotifyPending, r.NotifiedState = "", n
+	e.save(ctx, r)
+}
+
+func isActive(state string) bool {
+	return state == store.AlertPending || state == store.AlertFiring || state == store.AlertFlapping
+}
+
+// pendingValid reports whether notification n still describes the row.
+func pendingValid(r *store.Alert, n string) bool {
+	switch n {
+	case NotifyFiring:
+		return r.BaseState == store.AlertFiring && r.State != store.AlertFlapping
+	case NotifyResolved:
+		return r.State == store.AlertResolved && (r.NotifiedState == NotifyFiring || r.NotifiedState == NotifyFlapping)
+	case NotifyFlapping:
+		return r.State == store.AlertFlapping
+	}
+	return false
+}
+
+// transition builds the notification. For a firing device_down that shadows
+// downstream devices it names every root-cause device and counts the shadowed
+// devices; group is then a key shared by the notices of the same outage, so
+// two parents failing in one pass produce one message.
+func (e *Evaluator) transition(_ time.Time, p *pass, r *store.Alert, n string) (t Transition, group string) {
+	t = Transition{AlertID: r.ID, Rule: r.Rule, State: n, Severity: r.Severity, Device: r.DeviceSlug, Reason: r.Component,
+		ClearedAt: r.ClearedAt, Summary: r.Summary, Description: r.Description}
+	if r.FiredAt != nil {
+		t.FiredAt = *r.FiredAt
+	}
+	if r.Rule != RuleDown || n != NotifyFiring {
+		return t, ""
+	}
+	below := e.g.ShadowedBelow(r.DeviceSlug, p.shadowed)
+	if len(below) == 0 {
+		return t, ""
+	}
+	t.Parents, t.SuppressedCount = e.rootCauses(p, r.DeviceSlug, below), len(below)
+	t.Summary = fmt.Sprintf("%s down: %d downstream devices suppressed", joinSlugs(t.Parents), t.SuppressedCount)
+	t.Description = fmt.Sprintf("%s is down. Down with it: %s. %d devices behind them are unreachable and their alerts are suppressed.",
+		r.DeviceSlug, joinSlugs(t.Parents), t.SuppressedCount)
+	return t, "outage|" + joinSlugs(t.Parents)
+}
+
+// rootCauses is root plus every down, not-shadowed parent of the devices root's outage shadows.
+func (e *Evaluator) rootCauses(p *pass, root string, below []string) []string {
+	set := map[string]bool{root: true}
+	for _, c := range below {
+		for _, par := range e.g.Parents(c) {
+			if p.down[par] && !p.shadowed[par] {
+				set[par] = true
+			}
+		}
+	}
+	out := make([]string, 0, len(set))
+	for s := range set {
+		out = append(out, s)
+	}
+	sort.Strings(out)
+	return out
+}

internal/reach/evaluator.go

@@ -0,0 +1,289 @@
+package reach
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/v10networks/nmon/internal/deps"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// Store is the part of the store the evaluator uses.
+type Store interface {
+	store.Alerts
+	ListDevices(ctx context.Context, f store.DeviceFilter, p store.Page) ([]store.Device, int64, error)
+	ListAllParents(ctx context.Context, relation string) ([]store.DeviceParent, error)
+}
+
+// Evaluator judges every device's reachability. Create it with New, call
+// Start once, then Run (or Evaluate on your own schedule).
+type Evaluator struct {
+	o   Options
+	log *slog.Logger
+	g   *deps.Graph
+
+	mu   sync.Mutex // guards devs and the signals and status inside
+	devs map[string]*device
+
+	evalMu      sync.Mutex // serialises Start, Refresh and Evaluate; guards rows and lastRefresh
+	rows        map[string]*store.Alert
+	lastRefresh time.Time
+}
+
+// New returns an Evaluator; it does nothing until Start.
+func New(o Options) *Evaluator {
+	o.defaults()
+	return &Evaluator{o: o, log: o.Log, g: deps.New(), devs: map[string]*device{}, rows: map[string]*store.Alert{}}
+}
+
+// Graph is the dependency graph the evaluator keeps refreshed, for the API's cycle check.
+func (e *Evaluator) Graph() *deps.Graph { return e.g }
+
+// Start loads persisted reachability alerts, so a restart neither re-fires nor
+// forgets, then loads devices and the graph.
+func (e *Evaluator) Start(ctx context.Context) error {
+	e.evalMu.Lock()
+	defer e.evalMu.Unlock()
+	rows, err := e.o.Store.ListActiveAlertStates(ctx)
+	if err != nil {
+		return fmt.Errorf("reach: load alerts: %w", err)
+	}
+	for i := range rows {
+		r := rows[i]
+		if r.Rule == RuleDown || r.Rule == RuleWarning {
+			e.rows[r.SeriesKey] = &r
+		}
+	}
+	return e.refreshLocked(ctx)
+}
+
+// Run evaluates every Interval until ctx ends.
+func (e *Evaluator) Run(ctx context.Context) error {
+	if err := e.Start(ctx); err != nil {
+		return err
+	}
+	t := e.o.Clock.NewTicker(e.o.Interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-t.C():
+			e.Evaluate(ctx)
+		}
+	}
+}
+
+// Refresh reloads devices and the dependency graph, for example after the API changes parents.
+func (e *Evaluator) Refresh(ctx context.Context) error {
+	e.evalMu.Lock()
+	defer e.evalMu.Unlock()
+	return e.refreshLocked(ctx)
+}
+
+func (e *Evaluator) refreshLocked(ctx context.Context) error {
+	var devs []store.Device
+	for off := int64(0); ; off += store.MaxLimit {
+		page, total, err := e.o.Store.ListDevices(ctx, store.DeviceFilter{}, store.Page{Offset: off, Limit: store.MaxLimit})
+		if err != nil {
+			return fmt.Errorf("reach: list devices: %w", err)
+		}
+		devs = append(devs, page...)
+		if int64(len(devs)) >= total || len(page) == 0 {
+			break
+		}
+	}
+	links, err := e.o.Store.ListAllParents(ctx, store.RelNetwork)
+	if err != nil {
+		return fmt.Errorf("reach: list parents: %w", err)
+	}
+	e.g.Replace(deps.EdgesFrom(devs, links))
+	e.register(devs)
+	e.retireOrphans(ctx)
+	e.lastRefresh = e.o.Clock.Now()
+	return nil
+}
+
+// register syncs the device map with the inventory.
+func (e *Evaluator) register(devs []store.Device) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	now := e.o.Clock.Now()
+	live := map[string]bool{}
+	for i := range devs {
+		d := &devs[i]
+		if d.RetiredAt != nil || !d.Enabled {
+			continue
+		}
+		live[d.Slug] = true
+		info := Info{Slug: d.Slug, Kind: d.Kind}
+		if e.o.ICMPBlocked != nil {
+			info.ICMPBlocked = e.o.ICMPBlocked(d)
+		}
+		if cur := e.devs[d.Slug]; cur != nil {
+			info.CheckFails = cur.info.CheckFails
+			cur.info = info
+			continue
+		}
+		e.devs[d.Slug] = &device{info: info, registered: now, poll: d.PollState,
+			status: Status{State: Unknown, Last: Unknown, Since: now}}
+	}
+	for slug := range e.devs {
+		if !live[slug] {
+			delete(e.devs, slug)
+		}
+	}
+}
+
+// retireOrphans closes the rows of devices that are gone, retired or disabled.
+func (e *Evaluator) retireOrphans(ctx context.Context) {
+	e.mu.Lock()
+	var orphans []string
+	for _, r := range e.rows {
+		if e.devs[r.DeviceSlug] == nil {
+			orphans = append(orphans, r.DeviceSlug)
+		}
+	}
+	e.mu.Unlock()
+	for _, slug := range orphans {
+		e.retireRows(ctx, slug)
+	}
+}
+
+// SetCheckFails overrides how many consecutive failures take a check_only device down.
+func (e *Evaluator) SetCheckFails(slug string, n int) {
+	e.with(slug, func(d *device) { d.info.CheckFails = n })
+}
+
+// Status returns the live judgement for one device.
+func (e *Evaluator) Status(slug string) (Status, bool) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	d := e.devs[slug]
+	if d == nil {
+		return Status{}, false
+	}
+	return d.status, true
+}
+
+// Reachability implements api.ReachabilitySource: up, down, warning, unknown
+// or maintenance per slug. A degraded device shows its last known reachability
+// (the API shows the poll state separately); an unknown slug is unknown.
+func (e *Evaluator) Reachability(_ context.Context, devs []store.Device) map[string]string {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	out := make(map[string]string, len(devs))
+	for i := range devs {
+		out[devs[i].Slug] = string(Unknown)
+		d := e.devs[devs[i].Slug]
+		if d == nil {
+			continue
+		}
+		if s := d.status.State; s != Degraded {
+			out[devs[i].Slug] = string(s)
+		} else {
+			out[devs[i].Slug] = string(d.status.Last)
+		}
+	}
+	return out
+}
+
+// Evaluate runs one pass: judge every device, work out suppression, move the
+// alert rows and deliver what is due.
+func (e *Evaluator) Evaluate(ctx context.Context) {
+	e.evalMu.Lock()
+	defer e.evalMu.Unlock()
+	now := e.o.Clock.Now()
+	if now.Sub(e.lastRefresh) >= e.o.Refresh {
+		if err := e.refreshLocked(ctx); err != nil {
+			e.log.Warn("reach refresh failed", "err", err)
+		}
+	}
+	p := e.judge(now)
+	e.applyRows(ctx, now, p)
+}
+
+// pass is what one Evaluate computes once and shares across devices.
+type pass struct {
+	slugs    []string
+	view     map[string]devView
+	down     map[string]bool
+	shadowed map[string]bool
+	sent     map[string]bool // coalescing keys of group notifications already sent this pass
+}
+
+// devView is a device's state at the start of a pass.
+type devView struct {
+	display      State
+	last         State
+	reason       string
+	paused       bool // maintenance or degraded: alert rows are left alone
+	maintenance  bool
+	noICMP       bool // the ICMP probe is unsupported for this device
+	suppressedBy []string
+}
+
+func (p *pass) eff(n string) bool { return p.down[n] || p.shadowed[n] }
+
+// judge applies the policies to every device and records the outcome in its Status.
+func (e *Evaluator) judge(now time.Time) *pass {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	p := &pass{view: make(map[string]devView, len(e.devs)), down: map[string]bool{}, sent: map[string]bool{}}
+	for slug, d := range e.devs {
+		p.slugs = append(p.slugs, slug)
+		v := e.judgeOne(d, now)
+		p.view[slug] = v
+		p.down[slug] = v.last == Down
+	}
+	sort.Strings(p.slugs)
+	p.shadowed = e.g.Shadowed(func(n string) bool { return p.down[n] })
+	for _, slug := range p.slugs {
+		v := p.view[slug]
+		if v.last == Down {
+			if sup, by := e.g.Suppressed(slug, p.eff); sup {
+				v.suppressedBy = by
+				p.view[slug] = v
+			}
+		}
+		st := &e.devs[slug].status
+		st.SuppressedBy = v.suppressedBy
+	}
+	return p
+}
+
+func (e *Evaluator) judgeOne(d *device, now time.Time) devView {
+	maint := d.poll == PollMaintenance || (e.o.InMaintenance != nil && e.o.InMaintenance(d.info.Slug, now))
+	degraded := d.poll == PollDegraded || d.poll == PollBackoff
+	prev := d.status
+	if !degraded { // a backing-off poller gives no new evidence; the last verdict stands
+		if v := e.assess(d, now); !v.keep {
+			d.setLast(v.state, v.reason)
+		}
+	}
+	display := d.status.Last
+	reason := d.lastReason
+	switch {
+	case maint:
+		display, reason = Maintenance, ""
+	case degraded:
+		display, reason = Degraded, ""
+	case display != Warning:
+		reason = ""
+	}
+	d.status.State, d.status.Reason = display, reason
+	if prev.State != display || prev.Reason != reason {
+		d.status.Since = now
+	}
+	return devView{display: display, last: d.status.Last, reason: d.lastReason, paused: maint || degraded, maintenance: maint,
+		noICMP: d.icmp.unsupported}
+}
+
+func (d *device) setLast(s State, reason string) {
+	d.status.Last = s
+	d.lastReason = reason
+}

internal/reach/harness_test.go

@@ -0,0 +1,173 @@
+package reach
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/deps"
+	"github.com/v10networks/nmon/internal/selfmon"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+type fakeN struct {
+	mu   sync.Mutex
+	got  []Transition
+	fail error
+}
+
+func (n *fakeN) Notify(_ context.Context, t Transition) error {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if n.fail != nil {
+		return n.fail
+	}
+	n.got = append(n.got, t)
+	return nil
+}
+
+func (n *fakeN) take() []Transition {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	out := n.got
+	n.got = nil
+	return out
+}
+
+type harness struct {
+	t     *testing.T
+	st    store.Store
+	clk   *clock.Fake
+	n     *fakeN
+	ev    *Evaluator
+	opts  Options
+	slug  map[string]string // device name -> store slug
+	names map[string]string // store slug -> device name
+	maint func(slug string, at time.Time) bool
+}
+
+func newHarness(t *testing.T) *harness {
+	t.Helper()
+	clk := clock.NewFake(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC))
+	ctx := context.Background()
+	st, err := store.Open(ctx, store.DriverSQLite, filepath.Join(t.TempDir(), "inv.db"), nil, clk)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = st.Close() })
+	if err := st.Migrate(ctx); err != nil {
+		t.Fatal(err)
+	}
+	return &harness{t: t, st: st, clk: clk, n: &fakeN{}, slug: map[string]string{}, names: map[string]string{}}
+}
+
+// start creates a fresh Evaluator over the same database, as a process restart would.
+func (h *harness) start() {
+	h.t.Helper()
+	h.opts = Options{Store: h.st, Notifier: h.n, Clock: h.clk, Registry: selfmon.New(),
+		ICMPBlocked: func(d *store.Device) bool { return d.Profile == "icmp_blocked" },
+		InMaintenance: func(slug string, at time.Time) bool {
+			return h.maint != nil && h.maint(slug, at)
+		}}
+	h.ev = New(h.opts)
+	if err := h.ev.Start(context.Background()); err != nil {
+		h.t.Fatal(err)
+	}
+}
+
+func (h *harness) add(name, kind, profile string) {
+	h.t.Helper()
+	d := &store.Device{Name: name, Kind: kind, Profile: profile, Enabled: true}
+	if err := h.st.CreateDevice(context.Background(), d); err != nil {
+		h.t.Fatal(err)
+	}
+	h.slug[name], h.names[d.Slug] = d.Slug, name
+}
+
+func (h *harness) addTopology(topo *deps.Topology) {
+	h.t.Helper()
+	ctx := context.Background()
+	for _, d := range topo.Devices {
+		h.add(d.Slug, store.KindNetwork, "")
+	}
+	for _, d := range topo.Devices {
+		var ids []int64
+		for _, p := range d.Parents {
+			dev, _ := h.st.GetDeviceBySlug(ctx, h.slug[p])
+			ids = append(ids, dev.ID)
+		}
+		dev, _ := h.st.GetDeviceBySlug(ctx, h.slug[d.Slug])
+		if err := h.st.SetParents(ctx, dev.ID, store.RelNetwork, ids); err != nil {
+			h.t.Fatal(err)
+		}
+	}
+}
+
+// eval advances the clock and runs one pass.
+func (h *harness) eval(d time.Duration) {
+	h.clk.Advance(d)
+	h.ev.Evaluate(context.Background())
+}
+
+// cycle simulates one 15 s poll cycle of every network device: those in dead
+// fail ICMP and SNMP, the rest answer both.
+func (h *harness) cycle(dead ...string) {
+	now := h.clk.Now()
+	isDead := map[string]bool{}
+	for _, d := range dead {
+		isDead[d] = true
+	}
+	for name, slug := range h.slug {
+		r := ProbeOK
+		if isDead[name] {
+			r = ProbeFail
+		}
+		h.ev.ObserveICMP(slug, r, now)
+		h.ev.ObserveSNMP(slug, r, now)
+	}
+	h.eval(15 * time.Second)
+}
+
+func (h *harness) cycles(n int, dead ...string) {
+	for i := 0; i < n; i++ {
+		h.cycle(dead...)
+	}
+}
+
+// notes returns and clears the notifications as "device:rule:state" with names, sorted by arrival.
+func (h *harness) notes() []string {
+	var out []string
+	for _, t := range h.n.take() {
+		out = append(out, h.names[t.Device]+":"+t.Rule+":"+t.State)
+	}
+	return out
+}
+
+func (h *harness) status(name string) Status {
+	h.t.Helper()
+	s, ok := h.ev.Status(h.slug[name])
+	if !ok {
+		h.t.Fatalf("%s not tracked", name)
+	}
+	return s
+}
+
+func (h *harness) wantState(name string, want State) {
+	h.t.Helper()
+	if got := h.status(name).State; got != want {
+		h.t.Fatalf("%s state = %s, want %s", name, got, want)
+	}
+}
+
+func (h *harness) wantNotes(want ...string) {
+	h.t.Helper()
+	if got := strings.Join(h.notes(), ","); got != strings.Join(want, ",") {
+		h.t.Fatalf("notifications = [%s], want [%s]", got, strings.Join(want, ","))
+	}
+}
+
+var context0 = context.Background()

internal/reach/policy.go

@@ -0,0 +1,118 @@
+package reach
+
+import (
+	"time"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// verdict is a policy's judgement. keep means the signals do not settle it
+// (one SNMP failure, ICMP down for 20 s): the previous verdict stands.
+type verdict struct {
+	state  State
+	reason string
+	keep   bool
+}
+
+var hold = verdict{keep: true}
+
+func warn(reason string) verdict { return verdict{state: Warning, reason: reason} }
+
+// assess applies the health policy of the device's kind to its signals.
+func (e *Evaluator) assess(d *device, now time.Time) verdict {
+	var v verdict
+	switch d.info.Kind {
+	case store.KindVM, store.KindContainer:
+		v = e.assessVM(d, now)
+	case store.KindCheckOnly:
+		v = assessCheck(d)
+	default:
+		v = e.assessNetwork(d, now)
+	}
+	stale := now.Sub(maxTime(d.lastSignal, d.registered)) > e.o.NoData
+	if stale && (v.keep || v.state == Up || v.state == Unknown) {
+		return warn(ReasonNoData)
+	}
+	return v
+}
+
+func maxTime(a, b time.Time) time.Time {
+	if a.After(b) {
+		return a
+	}
+	return b
+}
+
+// assessNetwork: down needs ICMP failing for ICMPDown AND the last SNMPFails
+// polls failed. One signal alone is a warning. With icmp_blocked, or when the
+// ICMP probe is unsupported on this host, SNMP alone decides (and the
+// icmp_unsupported warning tells the operator so).
+func (e *Evaluator) assessNetwork(d *device, now time.Time) verdict {
+	snmpFailed := d.snmp.fails >= e.o.SNMPFails
+	snmpOK := d.snmp.seen && d.snmp.fails == 0
+	if d.info.ICMPBlocked || d.icmp.unsupported { // unsupported is unknown, so SNMP alone decides
+		return snmpOnly(snmpFailed, snmpOK)
+	}
+	icmpFailing, icmpFailed := d.icmp.failing(now, e.o.ICMPDown)
+	switch {
+	case icmpFailed && snmpFailed:
+		return verdict{state: Down}
+	case icmpFailed && snmpOK:
+		return warn(ReasonICMPFiltered)
+	case snmpFailed && !icmpFailing:
+		return warn(ReasonSNMPUnreachable)
+	case icmpFailing || d.snmp.fails > 0:
+		return hold // a failure that has not lasted long enough to judge
+	case d.icmp.ok || snmpOK:
+		return verdict{state: Up}
+	}
+	return verdict{state: Unknown}
+}
+
+func snmpOnly(failed, ok bool) verdict {
+	switch {
+	case failed:
+		return verdict{state: Down}
+	case ok:
+		return verdict{state: Up}
+	}
+	return hold
+}
+
+// assessVM follows Proxmox and push absence; ICMP and SNMP never decide.
+func (e *Evaluator) assessVM(d *device, now time.Time) verdict {
+	if d.vm.seen && d.vm.state != VMRunning {
+		return verdict{state: Down}
+	}
+	if !d.lastPush.IsZero() && now.Sub(d.lastPush) > e.o.PushAbsent {
+		return warn(ReasonPushAbsent)
+	}
+	if (d.vm.seen && d.vm.state == VMRunning) || !d.lastPush.IsZero() {
+		return verdict{state: Up}
+	}
+	return verdict{state: Unknown}
+}
+
+// assessCheck: down once the check has failed its configured count in a row.
+func assessCheck(d *device) verdict {
+	limit := d.info.CheckFails
+	if limit <= 0 {
+		limit = DefaultCheckFails
+	}
+	switch {
+	case d.chk.fails >= limit:
+		return verdict{state: Down}
+	case d.chk.fails > 0:
+		return hold
+	case d.chk.seen:
+		return verdict{state: Up}
+	}
+	return verdict{state: Unknown}
+}
+
+// failing reports whether ICMP is in an unbroken run of failures, and whether
+// that run has lasted at least limit. Unsupported is neither.
+func (s icmpSignal) failing(now time.Time, limit time.Duration) (failing, failed bool) {
+	failing = s.seen && !s.ok && !s.unsupported && !s.failSince.IsZero()
+	return failing, failing && now.Sub(s.failSince) >= limit
+}

internal/reach/policy_test.go

@@ -0,0 +1,267 @@
+package reach
+
+import (
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+func TestNetworkIsDownOnlyAfterICMP45sAndTwoSNMPFailures(t *testing.T) {
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	h.cycles(2, "sw") // 30 s of failure: neither signal is conclusive
+	h.wantState("sw", Unknown)
+	h.wantNotes()
+	h.cycle("sw") // the evaluation lands at 45 s
+	h.wantState("sw", Down)
+	h.wantNotes("sw:device_down:firing")
+	h.cycles(10, "sw")
+	h.wantNotes() // one notification, not one per pass
+	h.cycles(2)
+	h.wantState("sw", Up)
+	h.wantNotes("sw:device_down:resolved")
+}
+
+func TestSingleSignalFailureIsAWarningAnnouncedAfterTheHold(t *testing.T) {
+	h := newHarness(t)
+	h.add("icmp-only", store.KindNetwork, "")
+	h.add("snmp-only", store.KindNetwork, "")
+	h.start()
+	for i := 0; i < 5; i++ {
+		now := h.clk.Now()
+		h.ev.ObserveICMP(h.slug["icmp-only"], ProbeFail, now)
+		h.ev.ObserveSNMP(h.slug["icmp-only"], ProbeOK, now)
+		h.ev.ObserveICMP(h.slug["snmp-only"], ProbeOK, now)
+		h.ev.ObserveSNMP(h.slug["snmp-only"], ProbeFail, now)
+		h.eval(15 * time.Second)
+	}
+	if s := h.status("icmp-only"); s.State != Warning || s.Reason != ReasonICMPFiltered {
+		t.Fatalf("icmp-only = %+v", s)
+	}
+	if s := h.status("snmp-only"); s.State != Warning || s.Reason != ReasonSNMPUnreachable {
+		t.Fatalf("snmp-only = %+v", s)
+	}
+	h.wantNotes() // inside the hold nothing is announced
+	for i := 0; i < 22; i++ {
+		now := h.clk.Now()
+		h.ev.ObserveICMP(h.slug["icmp-only"], ProbeFail, now)
+		h.ev.ObserveSNMP(h.slug["icmp-only"], ProbeOK, now)
+		h.ev.ObserveICMP(h.slug["snmp-only"], ProbeOK, now)
+		h.ev.ObserveSNMP(h.slug["snmp-only"], ProbeFail, now)
+		h.eval(15 * time.Second)
+	}
+	got := map[string]bool{}
+	for _, n := range h.notes() {
+		got[n] = true
+	}
+	if !got["icmp-only:device_warning:firing"] || !got["snmp-only:device_warning:firing"] || len(got) != 2 {
+		t.Fatalf("notifications = %v", got)
+	}
+}
+
+func TestEscalationToDownSendsNoWarningNotice(t *testing.T) {
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	// ICMP fails first; SNMP only starts failing a minute later.
+	for i := 0; i < 4; i++ {
+		h.ev.ObserveICMP(h.slug["sw"], ProbeFail, h.clk.Now())
+		h.ev.ObserveSNMP(h.slug["sw"], ProbeOK, h.clk.Now())
+		h.eval(15 * time.Second)
+	}
+	h.wantState("sw", Warning)
+	h.cycles(3, "sw")
+	h.wantState("sw", Down)
+	h.wantNotes("sw:device_down:firing")
+	h.cycles(2)
+	h.wantNotes("sw:device_down:resolved")
+}
+
+func TestICMPBlockedProfileLetsSNMPDecideAlone(t *testing.T) {
+	h := newHarness(t)
+	h.add("fw", store.KindNetwork, "icmp_blocked")
+	h.start()
+	fail := func() {
+		h.ev.ObserveICMP(h.slug["fw"], ProbeFail, h.clk.Now())
+		h.ev.ObserveSNMP(h.slug["fw"], ProbeFail, h.clk.Now())
+		h.eval(15 * time.Second)
+	}
+	fail()
+	h.wantState("fw", Unknown)
+	fail()
+	h.wantState("fw", Down)
+	h.wantNotes("fw:device_down:firing")
+}
+
+func TestUnsupportedICMPLetsSNMPDecideAndWarnsOnce(t *testing.T) {
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	for i := 0; i < 2; i++ {
+		h.ev.ObserveICMP(h.slug["sw"], ProbeUnsupported, h.clk.Now())
+		h.ev.ObserveSNMP(h.slug["sw"], ProbeFail, h.clk.Now())
+		h.eval(15 * time.Second)
+	}
+	h.wantState("sw", Down)
+	h.wantNotes("sw:device_warning:firing", "sw:device_down:firing")
+	for i := 0; i < 6; i++ {
+		h.ev.ObserveICMP(h.slug["sw"], ProbeUnsupported, h.clk.Now())
+		h.ev.ObserveSNMP(h.slug["sw"], ProbeFail, h.clk.Now())
+		h.eval(15 * time.Second)
+	}
+	h.wantNotes() // once, not every pass
+	// SNMP answers: up, and the icmp_unsupported warning stays while ICMP stays unsupported.
+	h.ev.ObserveICMP(h.slug["sw"], ProbeUnsupported, h.clk.Now())
+	h.ev.ObserveSNMP(h.slug["sw"], ProbeOK, h.clk.Now())
+	h.eval(15 * time.Second)
+	h.wantState("sw", Up)
+	h.wantNotes("sw:device_down:resolved")
+	h.ev.ObserveICMP(h.slug["sw"], ProbeOK, h.clk.Now())
+	h.eval(15 * time.Second)
+	h.wantNotes("sw:device_warning:resolved")
+}
+
+func TestUnsupportedICMPWithoutSNMPStaysUnknown(t *testing.T) {
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	h.ev.ObserveICMP(h.slug["sw"], ProbeUnsupported, h.clk.Now())
+	h.eval(15 * time.Second)
+	h.wantState("sw", Unknown)
+}
+
+func TestVMFollowsProxmoxAndPushAbsenceNeverSNMP(t *testing.T) {
+	h := newHarness(t)
+	h.add("vm", store.KindVM, "")
+	h.add("pushvm", store.KindVM, "")
+	h.start()
+	vm, push := h.slug["vm"], h.slug["pushvm"]
+	for i := 0; i < 10; i++ { // SNMP and ICMP failing must not matter
+		h.ev.ObserveSNMP(vm, ProbeFail, h.clk.Now())
+		h.ev.ObserveICMP(vm, ProbeFail, h.clk.Now())
+		h.ev.ObserveVM(vm, VMRunning, h.clk.Now())
+		h.eval(15 * time.Second)
+	}
+	h.wantState("vm", Up)
+	h.ev.ObserveVM(vm, VMStopped, h.clk.Now())
+	h.eval(15 * time.Second)
+	h.wantState("vm", Down)
+	h.wantNotes("vm:device_down:firing")
+	h.ev.ObserveVM(vm, VMRunning, h.clk.Now())
+	h.eval(15 * time.Second)
+	h.wantNotes("vm:device_down:resolved")
+	h.ev.ObserveVM(vm, VMMissing, h.clk.Now())
+	h.eval(15 * time.Second)
+	h.wantState("vm", Down)
+
+	// A push-only VM (no Proxmox, ICMP off) disappears: warning, never down.
+	h.ev.ObservePush(push, h.clk.Now())
+	h.eval(15 * time.Second)
+	h.wantState("pushvm", Up)
+	h.eval(5*time.Minute + time.Second)
+	if s := h.status("pushvm"); s.State != Warning || s.Reason != ReasonPushAbsent {
+		t.Fatalf("pushvm = %+v", s)
+	}
+	h.eval(time.Hour)
+	if s := h.status("pushvm"); s.State == Down {
+		t.Fatal("push absence took a VM down")
+	}
+}
+
+func TestCheckOnlyDownAfterConfiguredFailures(t *testing.T) {
+	h := newHarness(t)
+	h.add("chk", store.KindCheckOnly, "")
+	h.add("chk1", store.KindCheckOnly, "")
+	h.start()
+	h.ev.SetCheckFails(h.slug["chk1"], 1)
+	for i := 1; i <= 3; i++ {
+		h.ev.ObserveCheck(h.slug["chk"], ProbeFail, h.clk.Now())
+		h.ev.ObserveCheck(h.slug["chk1"], ProbeFail, h.clk.Now())
+		h.eval(15 * time.Second)
+		if i < 3 {
+			h.wantState("chk", Unknown)
+		}
+	}
+	h.wantState("chk", Down)
+	h.wantState("chk1", Down)
+	h.ev.ObserveCheck(h.slug["chk"], ProbeOK, h.clk.Now())
+	h.eval(15 * time.Second)
+	h.wantState("chk", Up)
+}
+
+func TestNoDataWarningAfterFiveMinutesOfSilence(t *testing.T) {
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	h.cycle()
+	h.wantState("sw", Up)
+	h.eval(4 * time.Minute)
+	h.wantState("sw", Up)
+	h.eval(2 * time.Minute)
+	if s := h.status("sw"); s.State != Warning || s.Reason != ReasonNoData {
+		t.Fatalf("status = %+v", s)
+	}
+	h.cycle()
+	h.wantState("sw", Up)
+}
+
+func TestMaintenanceSuppressesEverythingInsideTheWindowOnly(t *testing.T) {
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	from := h.clk.Now().Add(time.Minute)
+	to := from.Add(30 * time.Minute)
+	h.maint = func(_ string, at time.Time) bool { return !at.Before(from) && at.Before(to) }
+	h.cycles(3, "sw") // 45 s, before the window: it would fire at 45 s
+	h.wantState("sw", Down)
+	h.wantNotes("sw:device_down:firing")
+
+	h2 := newHarness(t)
+	h2.add("sw", store.KindNetwork, "")
+	h2.start()
+	h2.maint = func(_ string, at time.Time) bool { return !at.Before(from) && at.Before(to) }
+	h2.eval(time.Minute)
+	h2.cycles(8, "sw")
+	h2.wantState("sw", Maintenance)
+	h2.wantNotes()
+	if got := h2.ev.Reachability(context0, []store.Device{{Slug: h2.slug["sw"]}})[h2.slug["sw"]]; got != "maintenance" {
+		t.Fatalf("reachability = %s", got)
+	}
+	h2.clk.Set(to.Add(time.Second))
+	h2.cycle("sw")
+	h2.wantState("sw", Down)
+	h2.wantNotes("sw:device_down:firing")
+}
+
+func TestPollMaintenanceStateAlsoCounts(t *testing.T) {
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	h.ev.ObservePoll(h.slug["sw"], PollMaintenance, h.clk.Now())
+	h.cycles(6, "sw")
+	h.wantState("sw", Maintenance)
+	h.wantNotes()
+}
+
+func TestBackoffKeepsLastKnownReachabilityAsDegraded(t *testing.T) {
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	h.cycles(3, "sw")
+	h.wantNotes("sw:device_down:firing")
+	h.ev.ObservePoll(h.slug["sw"], PollBackoff, h.clk.Now())
+	h.eval(20 * time.Minute) // no signals at all, far past no_data
+	s := h.status("sw")
+	if s.State != Degraded || s.Last != Down {
+		t.Fatalf("status = %+v, want degraded with last down", s)
+	}
+	if got := h.ev.Reachability(context0, []store.Device{{Slug: h.slug["sw"]}})[h.slug["sw"]]; got != "down" {
+		t.Fatalf("reachability = %s, want the last known state", got)
+	}
+	h.wantNotes()
+	h.ev.ObservePoll(h.slug["sw"], PollOK, h.clk.Now())
+	h.cycles(2)
+	h.wantNotes("sw:device_down:resolved")
+}

internal/reach/rows.go

@@ -0,0 +1,310 @@
+package reach
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// want is what a device's current verdict asks of one alert row.
+type want int
+
+const (
+	wantKeep     want = iota // the verdict does not settle it; leave the row alone
+	wantActive               // the condition holds
+	wantInactive             // the condition cleared; resolve and announce if it was announced
+	wantQuiet                // superseded by a worse condition; close without announcing
+)
+
+// target is one alert row a device can have: the down alert, or one warning reason.
+type target struct {
+	rule, reason string
+	hold         time.Duration // condition must last this long before it fires
+}
+
+func (t target) key(slug string) string { return t.rule + "|" + slug + "||" + t.reason }
+
+func (e *Evaluator) holdFor(reason string) time.Duration {
+	if reason == ReasonICMPFiltered || reason == ReasonSNMPUnreachable {
+		return e.o.WarnHold
+	}
+	return 0
+}
+
+// wants maps a verdict to what it asks of a row of the given rule and reason.
+func wants(v devView, rule, reason string) want {
+	if reason == ReasonICMPUnsupported { // independent of the verdict
+		if v.noICMP {
+			return wantActive
+		}
+		return wantInactive
+	}
+	if v.last == Unknown {
+		return wantKeep
+	}
+	switch rule {
+	case RuleDown:
+		if v.last == Down {
+			return wantActive
+		}
+		return wantInactive
+	default:
+		switch {
+		case v.last == Warning && v.reason == reason:
+			return wantActive
+		case v.last == Down:
+			return wantQuiet
+		}
+		return wantInactive
+	}
+}
+
+// applyRows moves every device's alert rows to match its verdict and delivers.
+func (e *Evaluator) applyRows(ctx context.Context, now time.Time, p *pass) {
+	byDev := map[string][]*store.Alert{}
+	for _, r := range e.rows {
+		byDev[r.DeviceSlug] = append(byDev[r.DeviceSlug], r)
+	}
+	for _, slug := range p.slugs {
+		v := p.view[slug]
+		seen := map[string]bool{}
+		for _, r := range byDev[slug] {
+			seen[r.SeriesKey] = true
+			t := target{rule: r.Rule, reason: r.Component, hold: e.holdFor(r.Component)}
+			e.step(ctx, now, p, slug, r, t, v)
+		}
+		if v.paused {
+			continue
+		}
+		if v.noICMP && !seen[target{rule: RuleWarning, reason: ReasonICMPUnsupported}.key(slug)] {
+			e.step(ctx, now, p, slug, nil, target{rule: RuleWarning, reason: ReasonICMPUnsupported}, v)
+		}
+		switch v.last {
+		case Down:
+			e.stepNew(ctx, now, p, slug, target{rule: RuleDown}, v, seen)
+		case Warning:
+			t := target{rule: RuleWarning, reason: v.reason, hold: e.holdFor(v.reason)}
+			e.stepNew(ctx, now, p, slug, t, v, seen)
+		}
+	}
+	// Rows of devices that are no longer tracked are retired by refresh; here only deliveries remain.
+}
+
+func (e *Evaluator) stepNew(ctx context.Context, now time.Time, p *pass, slug string, t target, v devView, seen map[string]bool) {
+	if !seen[t.key(slug)] {
+		e.step(ctx, now, p, slug, nil, t, v)
+	}
+}
+
+// step advances one row (nil when the incident does not exist yet) and delivers.
+func (e *Evaluator) step(ctx context.Context, now time.Time, p *pass, slug string, r *store.Alert, t target, v devView) {
+	if v.paused { // maintenance and degraded leave rows exactly as they are and hold deliveries
+		return
+	}
+	w := wants(v, t.rule, t.reason)
+	if r != nil && flapExpired(now, r) {
+		exitFlap(now, r)
+	}
+	if r == nil {
+		if w != wantActive {
+			return
+		}
+		r = e.newRow(ctx, now, slug, t)
+	}
+	switch w {
+	case wantActive:
+		activate(now, r, t.hold)
+	case wantInactive:
+		if !deactivate(now, r) {
+			e.dropPending(ctx, r)
+			return
+		}
+	case wantQuiet:
+		if closeQuiet(now, r) {
+			e.dropPending(ctx, r)
+			return
+		}
+	}
+	if !e.save(ctx, r) {
+		return
+	}
+	e.deliver(ctx, now, p, r)
+	if r.State == store.AlertResolved && r.NotifyPending == "" {
+		delete(e.rows, r.SeriesKey)
+	}
+}
+
+func (e *Evaluator) newRow(ctx context.Context, now time.Time, slug string, t target) *store.Alert {
+	r := &store.Alert{
+		Rule: t.rule, SeriesKey: t.key(slug), DeviceSlug: slug, Component: t.reason,
+		Severity: "critical", State: store.AlertPending, BaseState: store.AlertPending,
+		PendingSince: &now, FlapChanges: "[]",
+	}
+	if t.rule == RuleWarning {
+		r.Severity = "warning"
+	}
+	if prev, err := e.o.Store.LatestAlertBySeries(ctx, r.SeriesKey); err == nil {
+		r.FlapChanges = prev.FlapChanges // flap history survives across incidents
+	}
+	r.Summary, r.Description = describe(r)
+	e.rows[r.SeriesKey] = r
+	return r
+}
+
+func describe(r *store.Alert) (summary, desc string) {
+	if r.Rule == RuleDown {
+		return r.DeviceSlug + " is down", r.DeviceSlug + " is not reachable"
+	}
+	return r.DeviceSlug + " warning: " + r.Component, r.DeviceSlug + " reachability warning (" + r.Component + ")"
+}
+
+func (e *Evaluator) save(ctx context.Context, r *store.Alert) bool {
+	r.LastEvalAt = ptr(e.o.Clock.Now())
+	if err := e.o.Store.UpsertAlertState(ctx, r); err != nil {
+		e.log.Error("persist reachability alert", "key", r.SeriesKey, "err", err)
+		return false
+	}
+	return true
+}
+
+func (e *Evaluator) dropPending(ctx context.Context, r *store.Alert) {
+	if r.ID != 0 {
+		if err := e.o.Store.DeletePendingAlert(ctx, r.ID); err != nil {
+			e.log.Error("drop pending reachability alert", "key", r.SeriesKey, "err", err)
+		}
+	}
+	delete(e.rows, r.SeriesKey)
+}
+
+// retireRows closes the rows of a device that left the inventory without announcing anything.
+func (e *Evaluator) retireRows(ctx context.Context, slug string) {
+	for k, r := range e.rows {
+		if r.DeviceSlug != slug {
+			continue
+		}
+		r.State, r.BaseState, r.NotifyPending, r.FlappingUntil = store.AlertRetired, store.AlertRetired, "", nil
+		r.ClearedAt = ptr(e.o.Clock.Now())
+		e.save(ctx, r)
+		delete(e.rows, k)
+	}
+}
+
+func ptr(t time.Time) *time.Time { return &t }
+
+// activate moves a row toward firing: after its hold, the condition fires.
+func activate(now time.Time, r *store.Alert, hold time.Duration) {
+	switch r.BaseState {
+	case store.AlertFiring:
+		return
+	case store.AlertResolved: // inside a flapping period: the condition returned
+		r.BaseState, r.PendingSince, r.ClearedAt = store.AlertPending, ptr(now), nil
+	}
+	if r.PendingSince == nil {
+		r.PendingSince = ptr(now)
+	}
+	if now.Sub(*r.PendingSince) >= hold {
+		fire(now, r)
+	}
+}
+
+// deactivate resolves a firing row. It returns false when the row was only
+// pending outside a flapping period and should be deleted.
+func deactivate(now time.Time, r *store.Alert) bool {
+	switch r.BaseState {
+	case store.AlertPending:
+		if !inFlap(now, r) {
+			return false
+		}
+		r.BaseState, r.ClearedAt = store.AlertResolved, ptr(now)
+	case store.AlertFiring:
+		resolve(now, r)
+	}
+	return true
+}
+
+// closeQuiet ends a warning that a worse condition replaced. It returns true
+// when the row never fired and should be deleted instead of kept as history.
+func closeQuiet(now time.Time, r *store.Alert) bool {
+	if r.BaseState == store.AlertPending {
+		return true
+	}
+	if r.State != store.AlertResolved {
+		r.State, r.BaseState, r.ClearedAt = store.AlertResolved, store.AlertResolved, ptr(now)
+		r.FlappingUntil = nil
+	}
+	r.NotifyPending = ""
+	return false
+}
+
+func fire(now time.Time, r *store.Alert) {
+	r.BaseState, r.ClearedAt = store.AlertFiring, nil
+	if r.FiredAt == nil {
+		r.FiredAt = ptr(now)
+	}
+	changed(now, r, NotifyFiring)
+}
+
+func resolve(now time.Time, r *store.Alert) {
+	r.BaseState, r.ClearedAt = store.AlertResolved, ptr(now)
+	changed(now, r, NotifyResolved)
+}
+
+// changed records a firing or resolved change: it sets the visible state,
+// counts the change for flap damping and picks the notification. Entering
+// flapping replaces the change that tipped the count over; while flapping
+// nothing is sent.
+func changed(now time.Time, r *store.Alert, note string) {
+	if inFlap(now, r) {
+		r.State = store.AlertFlapping
+		return
+	}
+	cut := now.Add(-FlapWindow).UnixMilli()
+	var kept []int64
+	for _, t := range parseChanges(r.FlapChanges) {
+		if t > cut {
+			kept = append(kept, t)
+		}
+	}
+	kept = append(kept, now.UnixMilli())
+	if len(kept) > FlapMax {
+		r.State, r.FlappingUntil, r.NotifyPending = store.AlertFlapping, ptr(now.Add(FlapHold)), NotifyFlapping
+	} else {
+		r.State, r.NotifyPending = r.BaseState, note
+	}
+	b, _ := json.Marshal(kept)
+	r.FlapChanges = string(b)
+}
+
+func parseChanges(s string) []int64 {
+	var t []int64
+	_ = json.Unmarshal([]byte(s), &t)
+	return t
+}
+
+func inFlap(now time.Time, r *store.Alert) bool {
+	return r.FlappingUntil != nil && now.Before(*r.FlappingUntil)
+}
+
+func flapExpired(now time.Time, r *store.Alert) bool {
+	return r.FlappingUntil != nil && !now.Before(*r.FlappingUntil)
+}
+
+// exitFlap ends a flapping period: the row takes its underlying state and one
+// notification with that state goes out (a non-firing base reads as resolved).
+func exitFlap(now time.Time, r *store.Alert) {
+	r.FlappingUntil, r.FlapChanges = nil, "[]"
+	if r.BaseState == store.AlertFiring {
+		r.State, r.NotifyPending = store.AlertFiring, NotifyFiring
+		return
+	}
+	r.BaseState, r.State = store.AlertResolved, store.AlertResolved
+	if r.ClearedAt == nil {
+		r.ClearedAt = ptr(now)
+	}
+	r.PendingSince, r.NotifyPending = nil, NotifyResolved
+}
+
+func joinSlugs(s []string) string { return strings.Join(s, ", ") }

internal/reach/signals.go

@@ -0,0 +1,140 @@
+package reach
+
+import "time"
+
+// device is the evaluator's memory of one device: raw signals, no judgement.
+// All fields are guarded by Evaluator.mu.
+type device struct {
+	info       Info
+	registered time.Time
+	poll       string
+
+	icmp icmpSignal
+	snmp snmpSignal
+	vm   vmSignal
+	chk  checkSignal
+
+	lastPush   time.Time
+	lastSignal time.Time
+
+	status     Status
+	lastReason string // warning reason that goes with status.Last
+}
+
+type icmpSignal struct {
+	seen        bool
+	ok          bool
+	unsupported bool
+	failSince   time.Time
+}
+
+type snmpSignal struct {
+	seen  bool
+	fails int
+}
+
+type vmSignal struct {
+	seen  bool
+	state VMState
+}
+
+type checkSignal struct {
+	seen  bool
+	fails int
+}
+
+var _ Observer = (*Evaluator)(nil)
+
+// touch records data arriving from a real probe; unsupported probes carry no data.
+func (d *device) touch(at time.Time) {
+	if at.After(d.lastSignal) {
+		d.lastSignal = at
+	}
+}
+
+// with runs fn on the named device under the lock; an unknown slug is ignored
+// (the event raced a refresh, or the device is retired).
+func (e *Evaluator) with(slug string, fn func(d *device)) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if d := e.devs[slug]; d != nil {
+		fn(d)
+	}
+}
+
+// ObserveICMP records an ICMP result. Failure time is counted from the first
+// failure of an unbroken run.
+func (e *Evaluator) ObserveICMP(slug string, r Probe, at time.Time) {
+	e.with(slug, func(d *device) {
+		s := &d.icmp
+		s.seen = true
+		switch r {
+		case ProbeUnsupported:
+			s.unsupported, s.ok, s.failSince = true, false, time.Time{}
+		case ProbeOK:
+			s.unsupported, s.ok, s.failSince = false, true, time.Time{}
+			d.touch(at)
+		case ProbeFail:
+			if s.ok || s.failSince.IsZero() || s.unsupported {
+				s.failSince = at
+			}
+			s.unsupported, s.ok = false, false
+			d.touch(at)
+		}
+	})
+}
+
+// ObserveSNMP records one SNMP poll result; failures are counted consecutively.
+func (e *Evaluator) ObserveSNMP(slug string, r Probe, at time.Time) {
+	if r == ProbeUnsupported {
+		return
+	}
+	e.with(slug, func(d *device) {
+		d.snmp.seen = true
+		if r == ProbeOK {
+			d.snmp.fails = 0
+		} else {
+			d.snmp.fails++
+		}
+		d.touch(at)
+	})
+}
+
+// ObserveVM records the Proxmox state of a VM or container.
+func (e *Evaluator) ObserveVM(slug string, s VMState, at time.Time) {
+	e.with(slug, func(d *device) {
+		d.vm = vmSignal{seen: true, state: s}
+		d.touch(at)
+	})
+}
+
+// ObserveCheck records a check_only device's check result.
+func (e *Evaluator) ObserveCheck(slug string, r Probe, at time.Time) {
+	if r == ProbeUnsupported {
+		return
+	}
+	e.with(slug, func(d *device) {
+		d.chk.seen = true
+		if r == ProbeOK {
+			d.chk.fails = 0
+		} else {
+			d.chk.fails++
+		}
+		d.touch(at)
+	})
+}
+
+// ObservePush records that a push sample arrived for the device.
+func (e *Evaluator) ObservePush(slug string, at time.Time) {
+	e.with(slug, func(d *device) {
+		if at.After(d.lastPush) {
+			d.lastPush = at
+		}
+		d.touch(at)
+	})
+}
+
+// ObservePoll records the poller's state for the device.
+func (e *Evaluator) ObservePoll(slug, state string, _ time.Time) {
+	e.with(slug, func(d *device) { d.poll = state })
+}

internal/reach/topology_test.go

@@ -0,0 +1,212 @@
+package reach
+
+import (
+	"context"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/deps"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+const redundant = "../../testdata/topologies/redundant.yaml"
+
+func redundantHarness(t *testing.T) (*harness, *deps.Topology) {
+	t.Helper()
+	topo, err := deps.LoadTopology(redundant)
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := newHarness(t)
+	h.addTopology(topo)
+	h.start()
+	h.cycles(2) // everything answers
+	h.wantNotes()
+	return h, topo
+}
+
+// below lists the devices of the redundant topology that sit behind both cores.
+func below(topo *deps.Topology) []string {
+	var out []string
+	for _, d := range topo.Devices {
+		if d.Role != "core" {
+			out = append(out, d.Slug)
+		}
+	}
+	return out
+}
+
+func TestOneCoreDownSuppressesNothingAndNotifiesOnce(t *testing.T) {
+	h, _ := redundantHarness(t)
+	h.cycles(3, "core-1")
+	got := h.n.take()
+	if len(got) != 1 || h.names[got[0].Device] != "core-1" || got[0].Rule != RuleDown || got[0].State != NotifyFiring {
+		t.Fatalf("notifications = %+v", got)
+	}
+	if got[0].SuppressedCount != 0 || len(got[0].Parents) != 0 {
+		t.Fatalf("a redundant core must not suppress anything: %+v", got[0])
+	}
+	if s := h.status("dist-1"); s.State != Up {
+		t.Fatalf("dist-1 = %+v", s)
+	}
+}
+
+func TestBothCoresInDifferentCyclesGiveOneNotificationNamingBoth(t *testing.T) {
+	h, topo := redundantHarness(t)
+	h.cycles(3, "core-1")
+	h.wantNotes("core-1:device_down:firing")
+	// Core-2 fails later, and the children behind them fail with it.
+	dead := append([]string{"core-1", "core-2"}, below(topo)...)
+	h.cycles(3, dead...)
+	got := h.n.take()
+	if len(got) != 1 {
+		t.Fatalf("want exactly one notification for the second core, got %d: %+v", len(got), got)
+	}
+	n := got[0]
+	var names []string
+	for _, p := range n.Parents {
+		names = append(names, h.names[p])
+	}
+	if h.names[n.Device] != "core-2" || strings.Join(names, ",") != "core-1,core-2" && strings.Join(names, ",") != "core-2,core-1" {
+		t.Fatalf("notification = %+v names=%v", n, names)
+	}
+	if n.SuppressedCount != 22 {
+		t.Fatalf("suppressed count = %d, want 22", n.SuppressedCount)
+	}
+	// The children are down but suppressed: recorded with suppressed_by, silent.
+	rows, err := h.st.ListActiveAlertStates(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	suppressed := 0
+	for _, r := range rows {
+		if r.Rule == RuleDown && r.SuppressedBy != "" {
+			suppressed++
+		}
+	}
+	if suppressed != 22 {
+		t.Fatalf("rows with suppressed_by = %d, want 22", suppressed)
+	}
+	if s := h.status("dist-1"); s.State != Down || len(s.SuppressedBy) != 2 {
+		t.Fatalf("dist-1 = %+v", s)
+	}
+	h.cycles(10, dead...)
+	h.wantNotes()
+}
+
+func TestBothCoresInOnePassGiveOneNotification(t *testing.T) {
+	h, topo := redundantHarness(t)
+	h.cycles(3, append([]string{"core-1", "core-2"}, below(topo)...)...)
+	got := h.n.take()
+	if len(got) != 1 || got[0].SuppressedCount != 22 || len(got[0].Parents) != 2 {
+		t.Fatalf("notifications = %+v", got)
+	}
+}
+
+func TestCoreRecoveryUnsuppressesStillDownChildrenWhichNotifyIndividually(t *testing.T) {
+	h, topo := redundantHarness(t)
+	dead := append([]string{"core-1", "core-2"}, below(topo)...)
+	h.cycles(3, dead...)
+	h.n.take()
+	// core-2 comes back; its children are still down.
+	var still []string
+	for _, d := range dead {
+		if d != "core-2" {
+			still = append(still, d)
+		}
+	}
+	h.cycles(3, still...)
+	got := h.notes()
+	sort.Strings(got)
+	want := []string{"core-2:device_down:resolved", "dist-1:device_down:firing", "dist-2:device_down:firing"}
+	if strings.Join(got, ",") != strings.Join(want, ",") {
+		t.Fatalf("notifications = %v, want %v (access switches stay suppressed behind their dist)", got, want)
+	}
+}
+
+func TestChildThatRecoversWhileSuppressedNeverNotifies(t *testing.T) {
+	h, topo := redundantHarness(t)
+	dead := append([]string{"core-1", "core-2"}, below(topo)...)
+	h.cycles(3, dead...)
+	h.n.take()
+	h.cycles(2, "core-1", "core-2")
+	h.wantNotes()
+	h.cycles(2)
+	got := h.notes()
+	sort.Strings(got)
+	if strings.Join(got, ",") != "core-1:device_down:resolved,core-2:device_down:resolved" {
+		t.Fatalf("notifications = %v", got)
+	}
+}
+
+func TestRestartOverTheSameDatabaseNeitherRefiresNorForgets(t *testing.T) {
+	h, _ := redundantHarness(t)
+	h.cycles(3, "core-1")
+	h.wantNotes("core-1:device_down:firing")
+	h.start()
+	h.cycles(4, "core-1")
+	h.wantNotes()
+	h.start()
+	h.cycles(2) // recovery after the restart still resolves the old incident
+	h.wantNotes("core-1:device_down:resolved")
+}
+
+func TestRestartDeliversANotificationThatWasNotSent(t *testing.T) {
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	h.n.fail = context.DeadlineExceeded
+	h.cycles(3, "sw")
+	h.wantNotes()
+	h.n.fail = nil
+	h.start()
+	h.cycles(1, "sw")
+	h.wantNotes("sw:device_down:firing")
+}
+
+func TestFlappingGivesOneNoticeInAndOneOut(t *testing.T) {
+	h := newHarness(t)
+	h.add("vm", store.KindVM, "")
+	h.start()
+	toggle := func(s VMState) {
+		h.ev.ObserveVM(h.slug["vm"], s, h.clk.Now())
+		h.eval(5 * time.Second)
+	}
+	for i := 0; i < 3; i++ {
+		toggle(VMStopped)
+		toggle(VMRunning)
+	}
+	h.wantNotes("vm:device_down:firing", "vm:device_down:resolved", "vm:device_down:firing", "vm:device_down:resolved",
+		"vm:device_down:firing", "vm:device_down:flapping")
+	// Fifteen minutes of further flapping and recovery stays silent...
+	for i := 0; i < 20; i++ {
+		toggle(VMStopped)
+		toggle(VMRunning)
+	}
+	h.wantNotes()
+	// ...until the hold ends: one notice out.
+	for i := 0; i < 200; i++ {
+		h.ev.ObserveVM(h.slug["vm"], VMRunning, h.clk.Now())
+		h.eval(5 * time.Second)
+	}
+	h.wantNotes("vm:device_down:resolved")
+}
+
+func TestReachabilityMapMatchesTheAPIInterface(t *testing.T) {
+	var _ interface {
+		Reachability(ctx context.Context, devs []store.Device) map[string]string
+	} = (*Evaluator)(nil)
+	var _ Observer = (*Evaluator)(nil)
+	h := newHarness(t)
+	h.add("sw", store.KindNetwork, "")
+	h.start()
+	h.cycles(3, "sw")
+	devs, _, _ := h.st.ListDevices(context.Background(), store.DeviceFilter{}, store.Page{Limit: 10})
+	devs = append(devs, store.Device{Slug: "ghost"})
+	m := h.ev.Reachability(context.Background(), devs)
+	if m[h.slug["sw"]] != "down" || m["ghost"] != "unknown" {
+		t.Fatalf("map = %v", m)
+	}
+}

internal/reach/types.go

@@ -0,0 +1,135 @@
+// Package reach decides whether each device is reachable. It consumes events
+// (ICMP and SNMP results, Proxmox VM state, check results, push samples),
+// applies the health policy of the device's kind, suppresses children whose
+// every parent is down, and persists the transitions as alerts. It never
+// queries a metric store, so it keeps working when both are stopped.
+package reach
+
+import (
+	"context"
+	"time"
+)
+
+// State is a device's reachability.
+type State string
+
+// Reachability states. The first five are the values the API shows.
+const (
+	Up          State = "up"
+	Down        State = "down"
+	Warning     State = "warning"
+	Unknown     State = "unknown"
+	Maintenance State = "maintenance"
+	// Degraded means the poller is backing off or degraded; the last known
+	// reachability is kept (Status.Last) and no alert changes.
+	Degraded State = "degraded"
+)
+
+// Warning reasons.
+const (
+	ReasonICMPFiltered    = "icmp_filtered"
+	ReasonSNMPUnreachable = "snmp_unreachable"
+	ReasonNoData          = "no_data"
+	ReasonPushAbsent      = "push_absent"
+	// ReasonICMPUnsupported is raised alongside the verdict while ICMP cannot be probed, so the operator sees SNMP alone is deciding.
+	ReasonICMPUnsupported = "icmp_unsupported"
+)
+
+// Alert rule names written to the alerts table.
+const (
+	RuleDown    = "device_down"
+	RuleWarning = "device_warning"
+)
+
+// Probe is the outcome of one ICMP, SNMP or check attempt.
+type Probe int
+
+// Probe outcomes. Unsupported (ICMP without privilege, say) is never a failure.
+const (
+	ProbeOK Probe = iota
+	ProbeFail
+	ProbeUnsupported
+)
+
+// VMState is what Proxmox reports about a VM or container.
+type VMState int
+
+// VM states.
+const (
+	VMRunning VMState = iota
+	VMStopped
+	VMMissing
+)
+
+// Poll states the poller reports through ObservePoll (they match store.Poll*).
+const (
+	PollOK          = "ok"
+	PollDegraded    = "degraded"
+	PollBackoff     = "backoff"
+	PollMaintenance = "maintenance"
+)
+
+// Observer is what collection paths call. The poller, Proxmox sync, checks and
+// the ingest path hold this interface, not the Evaluator. All methods are safe
+// for concurrent use, cheap, and never block on the database.
+type Observer interface {
+	ObserveICMP(device string, r Probe, at time.Time)
+	ObserveSNMP(device string, r Probe, at time.Time)
+	ObserveVM(device string, s VMState, at time.Time)
+	ObserveCheck(device string, r Probe, at time.Time)
+	ObservePush(device string, at time.Time)
+	// ObservePoll reports the poller's state for the device (PollOK, PollDegraded, PollBackoff, PollMaintenance).
+	ObservePoll(device, state string, at time.Time)
+}
+
+// Status is the live judgement for one device.
+type Status struct {
+	State  State
+	Reason string // warning reason, or ""
+	// Last is the underlying reachability (up, down, warning, unknown) even
+	// while State reads maintenance or degraded.
+	Last State
+	// SuppressedBy names the parents that explain a down device, or nil.
+	SuppressedBy []string
+	Since        time.Time
+}
+
+// Transition is one notification wanted. It mirrors alert.Transition so the
+// app can feed both into notify.Queue; the incident ID is
+// "<AlertID>-<FiredAt unix>".
+type Transition struct {
+	AlertID               int64
+	Rule, State, Severity string
+	Device, Reason        string
+	FiredAt               time.Time
+	ClearedAt             *time.Time
+	Summary, Description  string
+	// Parents names the down devices that account for SuppressedCount
+	// downstream devices (the notifying device included). Empty when nothing
+	// is suppressed behind it.
+	Parents         []string
+	SuppressedCount int
+}
+
+// Notification states (same strings as internal/alert).
+const (
+	NotifyFiring   = "firing"
+	NotifyResolved = "resolved"
+	NotifyFlapping = "flapping"
+)
+
+// Notifier receives transitions. A non-nil error means "not delivered"; the
+// evaluator retries on its next pass. The app adapts it to notify.Queue.
+type Notifier interface {
+	Notify(ctx context.Context, t Transition) error
+}
+
+// Info is what the evaluator needs to know about a device beyond its signals.
+type Info struct {
+	Slug string
+	Kind string // store.Kind*
+	// ICMPBlocked is the profile's snmp.icmp_blocked: SNMP alone decides.
+	ICMPBlocked bool
+	// CheckFails is how many consecutive check failures mean down (0 = default 3).
+	CheckFails int
+}

internal/snmp/snmp.go

@@ -404,6 +404,7 @@ func (s *Session) do(ctx context.Context, typ pduType, f func() (*gosnmp.SnmpPac
 	}
 	if err != nil {
 		if errors.Is(err, gosnmp.ErrMaxRetriesExceeded) || strings.Contains(err.Error(), "timeout") {
+			selfmon.Default.Counter("nmon_snmp_timeouts_total", "path", s.path).Inc() // feeds GET /poller/health
 			return nil, fmt.Errorf("%w: %w", ErrTimeout, err)
 		}
 		return nil, err

internal/store/parent_links.go

@@ -0,0 +1,21 @@
+package store
+
+import "context"
+
+// ParentLinks reads the whole dependency graph in one query, for the
+// reachability evaluator and the cycle check, which would otherwise issue one
+// ListParents per device.
+type ParentLinks interface {
+	// ListAllParents returns every parent link of one relation (empty means all), ordered by device then parent.
+	ListAllParents(ctx context.Context, relation string) ([]DeviceParent, error)
+}
+
+func (s *bunStore) ListAllParents(ctx context.Context, relation string) ([]DeviceParent, error) {
+	var rows []DeviceParent
+	q := s.db.NewSelect().Model(&rows)
+	if relation != "" {
+		q = q.Where("relation = ?", relation)
+	}
+	err := q.OrderExpr("device_id, parent_id, relation").Scan(ctx)
+	return rows, mapErr(err)
+}