Fix two integration test failures seen in CI

TestRunReturnsStartError expected a start with no TLS flags to fail;
since the first-install wizard that start succeeds (install mode with
a self-signed certificate) and Run blocked until the 10 min timeout.
Use --tls-cert without --tls-key, which is always refused.

TestICMPStateMachine could advance the fake clock twice in one step on
a slow runner, firing an extra probe ([1 1 1 0 0 1 1]). Advance once,
only after the ICMP loop has re-armed its timer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017TuKgyKve2XLFtrrXWMi2w

.github/workflows/ci.yml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 jobs:
   lint:
     # Hard rule (docs/MAINTAINABILITY.md): any finding fails the build.
-    runs-on: ubuntu-latest
+    runs-on: debian-13
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
@@ -13,10 +13,7 @@ jobs:
       - run: cd web && npm ci
       - run: make lint
   test:
-    strategy:
-      matrix:
-        arch: [ubuntu-latest, ubuntu-24.04-arm]
-    runs-on: ${{ matrix.arch }}
+    runs-on: debian-13
     steps:
       - uses: actions/checkout@v4
         with: { fetch-depth: 0 }
@@ -30,14 +27,14 @@ jobs:
       - run: make test-v6only
       - run: make build && file nmon | grep -q 'statically linked'
   postgres:
-    runs-on: ubuntu-latest
+    runs-on: debian-13
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with: { go-version: stable }
       - run: make test-pg
   openapi:
-    runs-on: ubuntu-latest
+    runs-on: debian-13
     steps:
       - uses: actions/checkout@v4
         with: { fetch-depth: 0 }

.gitignore

@@ -14,3 +14,7 @@ coverage.out
 deploy/ansible/.mibs.tar.gz
 deploy/ansible/inventory.ini
 deploy/ansible/vars.yml
+
+# MCP Vector Search index directory
+.mcp-vector-search/
+/nmon-dev

cmd/nmon/devicepoll.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/v10networks/nmon/internal/cliout"
+	"github.com/v10networks/nmon/internal/poller"
+)
+
+func init() {
+	register("device", "operate on one device of a running nmon: nmon device poll <slug> [--test] [--json]", deviceCmd)
+}
+
+const devicePollUsage = "usage: nmon device poll <slug> [--test [--interval SECONDS]] [--json] --api URL --token TOKEN"
+
+// deviceCmd is "nmon device poll": ask the running nmon to poll one device now, or with --test to run a traced
+// test poll (the same as the device page's Test poll). The test never writes metrics.
+func deviceCmd(ctx context.Context, args []string) error {
+	if len(args) == 0 || args[0] != "poll" {
+		return errors.New(devicePollUsage)
+	}
+	slug, rest, err := leadingArg(args[1:])
+	if err != nil || slug == "" {
+		return errors.New(devicePollUsage)
+	}
+	var af apiFlags
+	var test, asJSON bool
+	var interval int
+	fs := flag.NewFlagSet("device poll", flag.ContinueOnError)
+	af.register(fs)
+	fs.BoolVar(&test, "test", false, "run a traced test poll now and print it; nothing is stored")
+	fs.IntVar(&interval, "interval", 0, "with --test: only the interval group with this cadence in seconds (default every group)")
+	fs.BoolVar(&asJSON, "json", false, "with --test: print the trace as JSON")
+	if err := fs.Parse(rest); err != nil {
+		return flagErr(err)
+	}
+	if err := af.validate(); err != nil {
+		return err
+	}
+	base := "/api/v1/devices/" + url.PathEscape(slug)
+	if !test {
+		resp, err := af.call(ctx, http.MethodPost, base+"/poll-now", nil)
+		if err != nil {
+			return err
+		}
+		_ = resp.Body.Close() // 202 with a one-field body that says nothing more
+		fmt.Printf("poll queued: %s will be polled now\n", slug)
+		return nil
+	}
+	resp, err := af.call(ctx, http.MethodPost, fmt.Sprintf("%s/poll-test?interval_seconds=%d", base, interval), nil)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }() // read in full below
+	if asJSON {
+		_, err = io.Copy(os.Stdout, resp.Body)
+		return err
+	}
+	var tr poller.Trace
+	if err := json.NewDecoder(resp.Body).Decode(&tr); err != nil {
+		return fmt.Errorf("decode the trace: %w", err)
+	}
+	return printTrace(os.Stdout, &tr)
+}
+
+// printTrace writes the summary, the requests and every OID that is not ok; --json has the full list.
+func printTrace(out io.Writer, tr *poller.Trace) error {
+	w := cliout.New(out)
+	w.Printf("test poll of %s: %s, SNMP %s, profile %s, %d ms\n", tr.Device, tr.Target, tr.SNMPVersion, tr.Profile, tr.DurationMS)
+	w.Printf("  requested %d, returned %d, missing %d, skipped %d\n", tr.Requested, tr.Returned, tr.Missing, tr.Skipped)
+	for _, g := range tr.Groups {
+		w.Printf("group %d s: %d ms, requested %d, returned %d, missing %d, would store %d\n",
+			g.IntervalSeconds, g.DurationMS, g.Requested, g.Returned, g.Missing, g.Samples)
+		if g.Error != "" {
+			w.Printf("  error: %s\n", g.Error)
+		}
+		for _, r := range g.Requests {
+			w.Printf("  request %d: %d varbinds from %s, %d ms, %s %s\n", r.Seq, r.Varbinds, r.FirstOID, r.DurationMS, r.Outcome, r.Error)
+		}
+		for _, r := range g.Results {
+			if r.Status != poller.StatusOK {
+				w.Printf("  %-18s %s %s %s %s\n", r.Status, r.OID, r.Series, r.Label, r.Detail)
+			}
+		}
+		if g.Truncated {
+			w.Printf("  (only %d of %d rows kept; use --json for the full list)\n", len(g.Results), g.ResultsTotal)
+		}
+	}
+	return w.Err()
+}

cmd/nmon/serve.go

@@ -51,7 +51,7 @@ func storeFlags(fs *flag.FlagSet, f *config.Flags) {
 
 func serve(ctx context.Context, args []string) error {
 	var f config.Flags
-	fs := newFlagSet("serve", "serve --data-dir DIR (--tls-cert F --tls-key F | --acme-host H | --insecure-http) [flags]")
+	fs := newFlagSet("serve", "serve --data-dir DIR [--tls-cert F --tls-key F | --acme-host H | --http-listen ADDR | --insecure-http] [flags]")
 	storeFlags(fs, &f)
 	fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile,
 		"SNMP credential key file; created on first run; must be outside --data-dir")
@@ -59,7 +59,13 @@ func serve(ctx context.Context, args []string) error {
 	fs.BoolVar(&f.InsecureHTTP, "insecure-http", false,
 		"development only: serve plain HTTP (insecure cookies); with no auth.oidc or auth.break_glass_file configured, "+
 			"authentication is off and every request is an admin")
-	fs.StringVar(&f.TLSCert, "tls-cert", "", "PEM certificate chain for HTTPS (reloaded on SIGHUP); needs --tls-key")
+	fs.StringVar(&f.HTTPListen, "http-listen", "",
+		"opt-in plain-HTTP listener for running behind a reverse proxy, e.g. [::1]:8080 (any interface allowed; empty = off); "+
+			"sign-in stays required (a fresh install opens the setup wizard first). Runs beside HTTPS, or alone when no TLS "+
+			"is configured. List the proxy in auth.trusted_proxies. "+
+			"Cannot be combined with --insecure-http")
+	fs.StringVar(&f.TLSCert, "tls-cert", "", "PEM certificate chain for HTTPS (reloaded on SIGHUP); needs --tls-key. "+
+		"With no TLS flag at all nmon serves HTTPS with a self-signed certificate kept in <data-dir>/tls")
 	fs.StringVar(&f.TLSKey, "tls-key", "", "PEM private key for --tls-cert")
 	fs.Func("acme-host", "obtain a Let's Encrypt certificate for this host name (repeatable); needs port 80 reachable", func(v string) error {
 		f.ACMEHosts = append(f.ACMEHosts, v)
@@ -77,7 +83,7 @@ func serve(ctx context.Context, args []string) error {
 	fs.StringVar(&f.ICMP, "icmp", config.ICMPAuto,
 		"ICMP probing: auto, or off for hosts where ICMP is not permitted (SNMP alone then decides if a device is down)")
 	fs.IntVar(&f.MaxInflight, "max-inflight", 512, "most SNMP polls in flight at once")
-	fs.DurationVar(&f.PollInterval, "poll-interval", 0, "dev: SNMP poll interval (default 60s)")
+	fs.DurationVar(&f.PollInterval, "poll-interval", 0, "dev: default SNMP poll interval (default 300s); a device's own interval overrides it")
 	fs.IntVar(&f.SNMPPort, "snmp-port", 0, "dev: UDP port of every device's agent (default 161)")
 	if err := fs.Parse(args); err != nil {
 		return flagErr(err)

deploy/ansible/README.md

@@ -6,7 +6,7 @@ Installs nmon as a systemd service on one Linux host (amd64 or arm64). Tested ag
 
     cd deploy/ansible
     cp inventory.example.ini inventory.ini   # your host
-    cp vars.example.yml vars.yml             # set nmon_breakglass_password at least
+    cp vars.example.yml vars.yml             # no sign-in set = the setup wizard runs on first open
     ansible-playbook site.yml -e @vars.yml
 
 The first play builds the release binary on your machine (`make build build-arm64`: needs Go, Node and network).
@@ -18,7 +18,8 @@ Pass `-e nmon_build=false` to reuse an existing `./nmon`. The second play instal
 - `config.yaml` rendered from your vars, and a hardened systemd unit (`systemctl reload nmon` sends SIGHUP to reload config and certificates)
 - `net.ipv4.ping_group_range` opened so nmon can ping without root
 
-Then open `https://<host>:8443/`. Without OIDC you are sent to `/auth/breakglass` (a browser password prompt; user `admin` by default).
+Then open `https://<host>:8443/`. With no sign-in in vars, nmon starts in install mode and the setup wizard asks for one
+(anyone who can reach the port can run it until it is done). Otherwise, without OIDC you are sent to `/auth/breakglass`.
 
 ## TLS (`nmon_tls_mode`)
 
@@ -35,6 +36,8 @@ Graphite (TCP 2003) go to nmon directly, not through the HTTP proxy.
 
 ## Sign-in
 
+- **Setup wizard:** leave `nmon_breakglass_password` and `nmon_oidc` unset. The play then writes `config.yaml` only
+  if it is missing, so later runs keep the sign-in the wizard saved (and ignore other config vars; edit those in the UI).
 - **OIDC:** set `nmon_oidc` (issuer, client_id, client_secret, admin_group). The secret is stored encrypted with
   `nmon secret set` and never written to `config.yaml`. Browsers go straight to the IdP; there is no login page.
 - **Break-glass:** `nmon_breakglass_password` is bcrypt-hashed on the host. Use it when the IdP is down. Five wrong

deploy/ansible/roles/nmon/defaults/main.yml

@@ -24,13 +24,16 @@ nmon_graphite_listen: "[::]:2003"
 nmon_icmp: auto
 nmon_timezone: UTC
 
-nmon_tls_mode: selfsigned          # selfsigned | files | acme | insecure
+nmon_tls_mode: selfsigned          # selfsigned | files | acme | insecure | none (none: plain HTTP only, needs nmon_http_listen)
+# Opt-in plain-HTTP listener for a reverse proxy, e.g. "[::1]:8080"; sign-in stays required. Empty = off.
+# Runs beside HTTPS, or alone with nmon_tls_mode=none. Not allowed with nmon_tls_mode=insecure.
+nmon_http_listen: ""
 nmon_server_name: "{{ inventory_hostname }}"
 nmon_acme_email: ""
 nmon_tls_extra_names: []            # more DNS names / IPs for the self-signed cert
 
 nmon_breakglass_user: admin
-nmon_breakglass_password: ""       # empty = no break-glass file (then OIDC must be configured)
+nmon_breakglass_password: ""       # empty (and no OIDC) = nmon starts in install mode: the setup wizard sets sign-in
 nmon_oidc: {}
 nmon_alerting: {}
 nmon_trusted_proxies: []

deploy/ansible/roles/nmon/tasks/main.yml

@@ -1,3 +1,10 @@
+- name: --http-listen and TLS mode agree
+  ansible.builtin.assert:
+    that:
+      - not (nmon_http_listen | length > 0 and nmon_tls_mode == 'insecure')
+      - nmon_tls_mode != 'none' or nmon_http_listen | length > 0
+    fail_msg: "nmon_http_listen cannot be combined with nmon_tls_mode=insecure, and nmon_tls_mode=none needs nmon_http_listen"
+
 - name: Check the platform (nmon ships linux/amd64 and linux/arm64)
   ansible.builtin.assert:
     that:
@@ -5,10 +12,9 @@
       - ansible_architecture in ["x86_64", "aarch64"]
     fail_msg: "nmon v1 supports linux/amd64 and linux/arm64 only"
 
-- name: Require a sign-in method when serving HTTPS (nmon refuses to start otherwise)
-  ansible.builtin.assert:
-    that: nmon_tls_mode == "insecure" or nmon_breakglass_password | length > 0 or nmon_oidc
-    fail_msg: "Set nmon_breakglass_password and/or nmon_oidc, or use nmon_tls_mode=insecure for a dev-open playground"
+- name: No sign-in in vars means first-install mode (the setup wizard picks the sign-in)
+  ansible.builtin.set_fact:
+    nmon_install_wizard: "{{ nmon_tls_mode != 'insecure' and nmon_breakglass_password | length == 0 and not nmon_oidc }}"
 
 - name: Install host packages (openssl for the self-signed cert, htpasswd for bcrypt, acl for become_user)
   ansible.builtin.package:
@@ -148,6 +154,8 @@
     owner: "{{ nmon_user }}"
     group: "{{ nmon_user }}"
     mode: "0640"
+    # The wizard writes its sign-in into config.yaml; never overwrite that with a template that has none.
+    force: "{{ not nmon_install_wizard | bool }}"
   notify: Reload nmon
 
 - name: Install the systemd unit
@@ -166,7 +174,7 @@
 
 - name: Wait for nmon to answer
   ansible.builtin.uri:
-    url: "{{ 'http' if nmon_tls_mode == 'insecure' else 'https' }}://localhost:{{ nmon_listen | regex_replace('^.*:', '') }}/healthz"
+    url: "{{ 'http' if nmon_tls_mode in ['insecure', 'none'] else 'https' }}://localhost:{{ (nmon_http_listen if nmon_tls_mode == 'none' else nmon_listen) | regex_replace('^.*:', '') }}/healthz"
     validate_certs: false
     status_code: 200
   register: nmon_health
@@ -177,6 +185,6 @@
 - name: Where to go next
   ansible.builtin.debug:
     msg:
-      - "nmon is up: {{ 'http' if nmon_tls_mode == 'insecure' else 'https' }}://{{ nmon_server_name }}:{{ nmon_listen | regex_replace('^.*:', '') }}/"
-      - "{{ 'Signs in via your IdP; break-glass at /auth/breakglass' if nmon_oidc else 'Sign in at /auth/breakglass as ' + nmon_breakglass_user }}"
+      - "nmon is up: {{ 'http' if nmon_tls_mode in ['insecure', 'none'] else 'https' }}://{{ nmon_server_name }}:{{ (nmon_http_listen if nmon_tls_mode == 'none' else nmon_listen) | regex_replace('^.*:', '') }}/"
+      - "{{ 'Signs in via your IdP; break-glass at /auth/breakglass' if nmon_oidc else ('Open it to run the setup wizard (first install), or sign in as configured there' if nmon_install_wizard | bool else 'Sign in at /auth/breakglass as ' + nmon_breakglass_user) }}"
       - "Back up the key file {{ nmon_key_file }} somewhere safe; without it a restore has no SNMP credentials."

deploy/ansible/roles/nmon/templates/nmon.service.j2

@@ -15,8 +15,13 @@ ExecStart=/usr/local/bin/nmon serve \
   --trap-listen {{ nmon_trap_listen }} \
   --graphite-listen {{ nmon_graphite_listen }} \
   --icmp {{ nmon_icmp }} \
+{% if nmon_http_listen | length > 0 %}
+  --http-listen {{ nmon_http_listen }} \
+{% endif %}
 {% if nmon_tls_mode == 'insecure' %}
   --insecure-http
+{% elif nmon_tls_mode == 'none' %}
+{# plain HTTP only, behind a reverse proxy: --http-listen above is the only listener #}
 {% elif nmon_tls_mode == 'acme' %}
   --acme-host {{ nmon_server_name }} --acme-email {{ nmon_acme_email }}
 {% else %}

deploy/ansible/vars.example.yml

@@ -8,9 +8,14 @@ nmon_server_name: "{{ inventory_hostname }}"   # a DNS name or an IP; it goes in
 # nmon_tls_key_src: files/nmon.key
 # nmon_acme_email: noc@example.net       # for tls_mode=acme; nmon answers http-01 on :80
 
+# Behind a reverse proxy: plain-HTTP listener (sign-in stays on). Add the proxy to auth.trusted_proxies (see docs/operations.md).
+# nmon_http_listen: "[::1]:8080"         # beside HTTPS; set nmon_tls_mode: none to serve only this listener
+
 # Break-glass local admin: the way in when the IdP is down (and the only sign-in until OIDC is set).
-nmon_breakglass_user: admin
-nmon_breakglass_password: "change-me-please"
+# The role writes this before nmon first starts, so the install is already "set up": nmon never enters install
+# mode (the first-install wizard that is open to anyone who can reach it). Leave it out only if you want the wizard.
+# nmon_breakglass_user: admin
+# nmon_breakglass_password: "change-me-please"
 
 # OIDC (optional). Unauthenticated browsers are redirected to the IdP; there is no login page.
 # nmon_oidc:

docs/DEVIATIONS.md

@@ -7,7 +7,7 @@ conscious, reviewed exception.
 |---|---|---|---|---|
 | 1 | Fixtures run on snmpsim (Python) via compose | `internal/simagent`, a Go agent that serves standard `.snmprec` files (v1/v2c/v3, dynamic counters, slow/flaky/toobig). `deploy/snmpsim-compose.yml` is kept for anyone using the reference simulator. | 2,000 agents for the loadtest in one process; no Python in CI; same file format, so real snmpsim recordings still work. | M1 |
 | 2 | "Two VictoriaMetrics executables are embedded" | One VictoriaMetrics single-node executable per architecture is embedded and run as two processes (hot and long) with different `-retentionPeriod`. | Both stores are the same program; embedding it twice would double the binary size for no gain. | M1 |
-| 3 | Libraries: huma, gosnmp, gnmic, go-oidc, bun; standard library elsewhere | Also: `oklog/ulid` (ULIDs), `gopkg.in/yaml.v3` (profiles, config), `golang.org/x/net` (unprivileged ICMP), `golang.org/x/crypto` (bcrypt, ACME), bun's SQLite (`modernc.org/sqlite`) and Postgres (`pgdriver`) drivers, `golang/snappy` + protobuf (remote write). | Each one fills a gap the standard library doesn't cover; each is small and widely used. | M1+ |
+| 3 | Libraries: huma, gosnmp, gnmic, go-oidc, bun; standard library elsewhere | Also: `oklog/ulid` (ULIDs), `gopkg.in/yaml.v3` (profiles, config), `golang.org/x/net` (unprivileged ICMP), `golang.org/x/crypto` (bcrypt, ACME), bun's SQLite (`modernc.org/sqlite`) and Postgres (`pgdriver`) drivers, `golang/snappy` + protobuf (remote write). Web: `react-markdown` + `remark-gfm` (the in-app `/help` page renders `docs/*.md` with tables and no raw HTML; hand-rolling a safe GFM renderer would be more code than the lazy chunk it replaces). | Each one fills a gap the standard library doesn't cover; each is small and widely used. | M1+ |
 | 4 | Integration tests use a VictoriaMetrics container | Integration tests run the VictoriaMetrics binary that `make fetch-vm` downloaded (the same pinned, checksummed one that is embedded). | Faster, and it tests the exact binary that ships. | M1 |
 | 5 | (not specified) M1 has no auth | Until M6, `nmon serve` requires `--insecure-http` and has no auth. | Auth is M6 in the build order. | M1–M5 |
 | 6 | "No MIB file is ever shipped in the repo or the binary"; vendor MIBs are the operator's to fetch | MIBs are bundled as a **sidecar** (user decision, 2026-10-22). All 4,865 LibreNMS MIBs are vendored under `mibs/` and shipped in the release tarball (installed to `/usr/share/nmon/mibs`). **None is embedded in the binary**, not even the IETF/IANA set. They are read via `--mibs-dir` (default `<data-dir>/mibs`, then `/usr/share/nmon/mibs`). The poller never reads a MIB. | Operators get `nmon profile new --mib` working out of the box while the binary stays small. Each MIB keeps its publisher's terms (see `mibs/README.md`). | M2 |
@@ -46,9 +46,18 @@ conscious, reviewed exception.
 | 39 | Spec: a profile that was matched stays ("non-generic is sticky"); no way to choose one by hand | Discovery re-matches **every unpinned device on every run**, so a newer or better profile is picked up, and at startup nmon queues a discovery for each unpinned device whose stored profile differs from what `Set.Match` picks now. An admin can pin a profile (`PUT /api/v1/devices/{slug}` with `profile`, or the Edit dialog); a pinned device keeps it, and a pinned profile that no longer exists gives generic plus the `profile_missing` warning as before. `profile: ""` returns to automatic. Migration 10 adds `devices.profile_pinned`; `GET /api/v1/profiles` lists the loaded profiles; `DeviceView` gains `profile_pinned` and `profile_match`. Profile changes are audited (`device.profile` from discovery, `device.edit` from the API). | The sticky rule meant a device matched once as generic stayed generic after an upgrade added its profile. | post-v1 (v1.0.1) |
 | 40 | Profiles shipped: the M2 set | Three more shipped profiles for the owner's gear: `ubiquiti-edgeswitch` (sysObjectID `1.3.6.1.4.1.4413` or sysDescr `^EdgeSwitch`, priority 30), `ubiquiti-unifi-switch` (sysDescr `USW`/`US-`/`USL`/`UniFi`, priority 40, so it beats edgeswitch for UniFi switches that report enterprise 4413), `vyos` (sysObjectID `1.3.6.1.4.1.44641` or sysDescr `VyOS`, extends `linux-netsnmp`, priority 20), and `ubiquiti-unifi-ap` (sysDescr `UAP`/`U6`/`U7`/`UAL`/`UK`/`E7`/`U-LTE`, UBNT-UniFi-MIB per-VAP clients, channel and tx power, per-radio channel utilisation; the MIB has no noise floor, and no per-radio channel or tx power, so those are per VAP). EdgeSwitch CPU is a text string and its temperature table has no label column, so neither is collected. | The owner's switches, routers and access points matched only `generic`. | post-v1 (v1.0.1) |
 | 41 | Default rules apply everywhere (spec §Alerting) | Alerts fire only where an **alert policy** binds a rule to a scope (groups plus device and port attribute matchers), stored in the `alert_policies` table (migration 12), edited through `/api/v1/alert-policies` and the Alerts > Alert policies tab, and applied live. With no policies nothing alerts: not metric rules, not `device_down`/`device_warning`, not system alarms (`system` policy, scope `all` only). An `ignore` policy beats an `alert` one at equal priority. Upgrading seeds nothing, so open alerts without a binding close silently as `retired` with no resolved notice. Supersedes #38's mechanism (the bell now only feeds the `alert_down` matcher) and makes `Rule.OptIn` a no-op. | Owner decision 2026-10-23: preprod alerted on everything, and a WiFi port that always sees errors is judged by the same rule as a fiber uplink where one error matters. | v1.1 |
-| 50 | The spec's collection table lists BGP-4-MIB but no profile fills it, and it names no MAC/ARP lookup | v1.3 (owner request 2026-10-24) adds both: BGP peers in the `routing` category (CISCO-BGP4 for cisco-ios/cisco-nxos, Juniper BGP4-V2, Arista BGP4V2, BGP4-MIB for the IPv4-only profiles that answer it) with a Routing tab, a BGP page, `GET /api/v1/bgp/peers` and the built-in `bgp_peer_down` rule; and a MAC/ARP/FDB lookup with a Find host page. Additive API: `/l2/search`, `/devices/{slug}/{l2,fdb,arp,l2/ports}`, `/interfaces/{id}/macs`, `/bgp/peers`; `/poller/health` gains `l2` and `notes`. | The owner wants to find where a host is plugged in and to see BGP sessions. | v1.3 |
-| 51 | (not specified) table walks run in discovery or not at all; the poll path never walks | A separate **L2 table job** (`internal/l2`, SNMP path label `l2` in `nmon_pdus_total`) walks BRIDGE-MIB, Q-BRIDGE-MIB, CISCO-VTP-MIB (per-VLAN context, profile flag `l2.per_vlan_context`, set on cisco-ios only) and IP-MIB neighbour tables every `l2.interval` (default 15m, minimum 5m) with its own concurrency cap (`l2.concurrency`, 4), a per-device deadline (`l2.device_timeout`, 2m) and the rollup job's yield rule (it pauses while the hot write queue is over half full). A device that times out or fails is skipped for the cycle, recorded on `/poller/health`, and never delays the others. Rows live in the inventory database (migration 14: `fdb_entries`, `arp_entries`, `vlans`, and `l2_walks`, one row per device holding the time of its last walk, which is how a row is "current"), are in `nmon backup` (JSONL export too) and are pruned daily by `last_seen` against `l2.keep` (90d). `interface_id` is the empty string, not NULL, when a bridge port has no ifIndex, because the unique key must not contain NULL. | The hot poll path must not walk tables; the history behind "where was this MAC last week" is inventory, not a metric. | v1.3 |
-| 52 | The component label of a table metric is a column of the table | Profiles may label rows by decoding the **row index** (`index_label.layout`, tokens `skip`, `skipinet`, `inet`, `skipaddr`, `addr`, `ipv4`, `peerindex`, `afisafi`), because BGP peer tables are indexed by an address. The label is the peer address (IPv6 compressed); a per-address-family prefix count is labelled `<peer>\|<afi>-<safi>`, for example `2001:db8::1\|ipv6-unicast`. Juniper's prefix table is keyed by a peer index, so its rows resolve through the `peer_index` attribute of the peer table (`index_label.via`). Peer AS, local AS, router id and description are component attributes (migration 11), never labels, so the SNMP label contract (`device` + `component`) is unchanged. Other profile additions: `enable_metrics` (switch on optional parent metrics), `wire: counter32` (CISCO-BGP4 declares accepted prefixes as a Counter32 but they are a count), attribute `format: ipbytes`, and `l2.per_vlan_context`. | A BGP peer is identified by its address, which only the index carries; the alternatives (a label column that does not exist, or labelling by row number) would break "each peer appears once" and the alert text. | v1.3 |
-| 53 | `bgp_peer_down` summary names the peer, AS and description | The rule's own summary names the device and the peer. The AS, router id and description are appended to the **notification** (Summary, Description and labels) when it is delivered, from the component attributes; the stored alert row keeps the plain summary. | The metric label contract allows only `device` and `component`, and the rule templates see only those; extending the evaluator's template data belongs to the v1.2 rules plumbing. | v1.3 |
-| 54 | (not specified) vendor of a MAC | An optional IEEE OUI registry (MA-L, MA-M, MA-S) shows the vendor of each MAC and flags locally administered (`local`, shown "Private / randomised") and multicast addresses. It is fetched weekly through the outbound allow-list (`oui.fetch: false` turns it off; `nmon oui import DIR` loads the CSVs offline), kept as `<data-dir>/oui/registry.gz` (checksummed, written atomically, in `nmon backup`), and looked up at query time by the longest prefix, so a refresh relabels history. Without a registry the vendor is empty, nothing fails, and `/poller/health` carries an `oui_unavailable` note (never an alert). The CSVs are public IEEE data and are not in the repository. | Owner request 2026-10-24: nice to have, never required. | v1.3 |
-| 55 | Lookup ranks edge ports "the port with the fewest MACs, then the most recent last_seen" | Within a MAC's locations the order is: edge ports before uplinks, then rows the device's last walk still saw before history, then fewest MACs, then most recent. A port is an uplink above `l2.edge_max_macs` MACs (8), with an LLDP neighbour (the lldpRemTable local port number is taken to be the ifIndex), or as a LAG (ifType 161). A port's MAC count is the MACs on it at or after the row's `last_seen`, so for a current row it is the port's MACs in the last walk. | A host that moved should show its new port first even when the old, now empty, port had fewer MACs. | v1.3 |
+| 42 | Alert rules are defined in code and `config.yaml` (spec §Alerting) | Rules are also defined in the database (`alert_rules`, migration 13) and edited through `/api/v1/alert-rules` and the Alerts > Alert rules tab, applied live on the next cycle. A custom rule is **builder only**: a metric rule names a catalogue metric (every pollable profile metric plus push-ingested names) with a transform, window, function, aggregate, comparison and threshold, and nmon generates the MetricsQL from fixed templates; an inventory rule ANDs one to five conditions over whitelisted device or port fields, evaluated in Go over store rows. No user text ever reaches a query. The shipped rules keep their names and become read-only "built-in" macros (three can be copied into the builder: `cpu_high`, `ups_on_battery`, `printer_supply_empty`). Comparison operators are now `> >= < <= == !=` everywhere. `alerting.rules` in `config.yaml` still works but is deprecated (the startup log says so); a custom rule cannot take the name of a built-in or config rule. `utilisation` (octets as a share of port speed) is not offered, because the port speed is not a stored series. | Owner decision 2026-10-23: the predefined rules are too constraining; admins need rules of their own from metrics and inventory fields. | post-v1 (v1.2) |
+| 43 | The spec's collection table lists BGP-4-MIB but no profile fills it, and it names no MAC/ARP lookup | v1.3 (owner request 2026-10-24) adds both: BGP peers in the `routing` category (CISCO-BGP4 for cisco-ios/cisco-nxos, Juniper BGP4-V2, Arista BGP4V2, BGP4-MIB for the IPv4-only profiles that answer it) with a Routing tab, a BGP page, `GET /api/v1/bgp/peers` and the built-in `bgp_peer_down` rule; and a MAC/ARP/FDB lookup with a Find host page. Additive API: `/l2/search`, `/devices/{slug}/{l2,fdb,arp,l2/ports}`, `/interfaces/{id}/macs`, `/bgp/peers`; `/poller/health` gains `l2` and `notes`. | The owner wants to find where a host is plugged in and to see BGP sessions. | v1.3 |
+| 44 | (not specified) table walks run in discovery or not at all; the poll path never walks | A separate **L2 table job** (`internal/l2`, SNMP path label `l2` in `nmon_pdus_total`) walks BRIDGE-MIB, Q-BRIDGE-MIB, CISCO-VTP-MIB (per-VLAN context, profile flag `l2.per_vlan_context`, set on cisco-ios only) and IP-MIB neighbour tables every `l2.interval` (default 15m, minimum 5m) with its own concurrency cap (`l2.concurrency`, 4), a per-device deadline (`l2.device_timeout`, 2m) and the rollup job's yield rule (it pauses while the hot write queue is over half full). A device that times out or fails is skipped for the cycle, recorded on `/poller/health`, and never delays the others. Rows live in the inventory database (migration 14: `fdb_entries`, `arp_entries`, `vlans`, and `l2_walks`, one row per device holding the time of its last walk, which is how a row is "current"), are in `nmon backup` (JSONL export too) and are pruned daily by `last_seen` against `l2.keep` (90d). `interface_id` is the empty string, not NULL, when a bridge port has no ifIndex, because the unique key must not contain NULL. | The hot poll path must not walk tables; the history behind "where was this MAC last week" is inventory, not a metric. | v1.3 |
+| 45 | The component label of a table metric is a column of the table | Profiles may label rows by decoding the **row index** (`index_label.layout`, tokens `skip`, `skipinet`, `inet`, `skipaddr`, `addr`, `ipv4`, `peerindex`, `afisafi`), because BGP peer tables are indexed by an address. The label is the peer address (IPv6 compressed); a per-address-family prefix count is labelled `<peer>\|<afi>-<safi>`, for example `2001:db8::1\|ipv6-unicast`. Juniper's prefix table is keyed by a peer index, so its rows resolve through the `peer_index` attribute of the peer table (`index_label.via`). Peer AS, local AS, router id and description are component attributes (migration 11), never labels, so the SNMP label contract (`device` + `component`) is unchanged. Other profile additions: `enable_metrics` (switch on optional parent metrics), `wire: counter32` (CISCO-BGP4 declares accepted prefixes as a Counter32 but they are a count), attribute `format: ipbytes`, and `l2.per_vlan_context`. | A BGP peer is identified by its address, which only the index carries; the alternatives (a label column that does not exist, or labelling by row number) would break "each peer appears once" and the alert text. | v1.3 |
+| 46 | `bgp_peer_down` summary names the peer, AS and description | The rule's own summary names the device and the peer. The AS, router id and description are appended to the **notification** (Summary, Description and labels) when it is delivered, from the component attributes; the stored alert row keeps the plain summary. | The metric label contract allows only `device` and `component`, and the rule templates see only those; extending the evaluator's template data belongs to the v1.2 rules plumbing. | v1.3 |
+| 47 | (not specified) vendor of a MAC | An optional IEEE OUI registry (MA-L, MA-M, MA-S) shows the vendor of each MAC and flags locally administered (`local`, shown "Private / randomised") and multicast addresses. It is fetched weekly through the outbound allow-list (`oui.fetch: false` turns it off; `nmon oui import DIR` loads the CSVs offline), kept as `<data-dir>/oui/registry.gz` (checksummed, written atomically, in `nmon backup`), and looked up at query time by the longest prefix, so a refresh relabels history. Without a registry the vendor is empty, nothing fails, and `/poller/health` carries an `oui_unavailable` note (never an alert). The CSVs are public IEEE data and are not in the repository. | Owner request 2026-10-24: nice to have, never required. | v1.3 |
+| 48 | Lookup ranks edge ports "the port with the fewest MACs, then the most recent last_seen" | Within a MAC's locations the order is: edge ports before uplinks, then rows the device's last walk still saw before history, then fewest MACs, then most recent. A port is an uplink above `l2.edge_max_macs` MACs (8), with an LLDP neighbour (the lldpRemTable local port number is taken to be the ifIndex), or as a LAG (ifType 161). A port's MAC count is the MACs on it at or after the row's `last_seen`, so for a current row it is the port's MACs in the last walk. | A host that moved should show its new port first even when the old, now empty, port had fewer MACs. | v1.3 |
+| 49 | SNMP interface counters are polled every 60 s; rate windows, rollup gap detection and reachability timers assume it | The default SNMP poll interval is **300 s** (LibreNMS-like; `store.DefaultPollInterval`), and an admin can set a different one per device (`poll_interval_s`, 0 or 30..3600, migration 15; empty = the default). The profile's 1m metrics follow it (halved on a device with unreliable 32-bit counters); 5 m and 15 m groups are unchanged and merge with a 300 s primary. Everything that was tied to the cadence follows the device's effective interval: graph, overview and p95 rate windows are two polls wide (at least 2 min; live rates use two default polls), the rollup's lookback and "thin window" test (a cadence of 5 min or slower predicts at most one sample per window, so a window is never thin, and a window with no sample is skipped as before), the `no_data` warning (silent for max(5 min, two polls)), the "unheard of for three polls is down" fallback, the up-series lookback (11 min), the dependency hold, and the `interface_errors` rule (10 min window). ICMP stays at 15 s. `nmon serve --poll-interval` still overrides the default for dev and tests. | Owner request 2026-10-24: 60 s polling of every port is more load than ISP-size fleets need; a per-device override keeps 1 min where it is wanted. Cost: a rollup window of a device polled slower than 5 min often holds no sample and is left empty, and graphs at the default have one point per 5 min. | post-v1.3 |
+| 50 | The API is served over HTTPS only; plain HTTP exists only as `--insecure-http` development mode (no sign-in) | An opt-in `--http-listen ADDR` serves the same UI and API over **plain HTTP with sign-in required**, beside HTTPS or alone when no TLS is configured. It is mutually exclusive with `--insecure-http`, never enables dev-open mode, and refuses to start with no OIDC or break-glass. Cookies are `Secure` on HTTPS and, on this listener, only behind a trusted proxy that sent `X-Forwarded-Proto: https`; client addresses come from `X-Forwarded-For` only from `auth.trusted_proxies`, and an empty list logs a WARN. | Owner request 2026-10-24: deployments sit behind a TLS-terminating reverse proxy in a DMZ or internal network, and double TLS to the backend adds nothing there. Cost: nmon cannot itself check the network between proxy and listener is private; the operator binds it to loopback or an internal address. | post-v1.4 |
+| 51 | Sign-in is configured before first start; nmon refuses to serve over TLS or `--http-listen` with none (Auth, DEVIATIONS 31 and 50) | **Install mode**: with no `auth.oidc`, no `auth.break_glass_file` and setup never completed, nmon serves a first-install wizard at `/setup` to **anyone who can reach it, unauthenticated, as an admin**. There is no setup token and no localhost restriction. Only the setup routes and what the wizard needs are served (UI, `GET /api/v1/setup`, settings form, notify presets/preview/test, device add/import and list, alert policies, profile list, secrets PUT, meta); every other API route answers 503 "setup not finished". A WARN at start and every 10 minutes says setup is open. Completing setup stores a one-way flag in the inventory (`setup_state`); it ends install mode permanently, switches authentication on without a restart, and cannot be undone from the web. An existing install with a sign-in method is marked completed at its first start, and deleting the break-glass file later does not reopen setup (the old refusal applies). `--insecure-http` dev-open mode is unchanged and never enters install mode. | Owner decision 2026-10-24: nmon is deployed in a DMZ or an internal network, and a token or localhost check would make a first install harder than the threat justifies. Cost: until setup completes, whoever reaches the port can create the administrator, so the port must not face an untrusted network before then. Ansible, which pre-configures sign-in, never enters install mode. | post-v1.4 |
+| 52 | Without a certificate flag nmon refuses to start ("HTTPS by default and has no certificate") | With none of `--tls-cert`/`--tls-key`, `--acme-host` or `--http-listen`, nmon generates a **self-signed** ECDSA P-256 certificate (names: host name, every non-loopback interface address incl. IPv6, `localhost`, `::1`, `127.0.0.1`; about 2 years; key 0600) in `<data-dir>/tls/`, reuses it on later starts, replaces it within 30 days of expiry, serves HTTPS on `--listen` and logs a WARN saying how to replace it. This applies to every start without TLS config, not only the first. User-provided certificates still reload on SIGHUP. `--http-listen` alone stays plain HTTP with no certificate. | Owner decision 2026-10-24: a first install must come up without a PKI. A browser warning is the price; the WARN names `--tls-cert`/`--tls-key` and `--acme-host` as the fix. | post-v1.4 |
+| 53 | (not specified) the default poll interval is a dev flag (`--poll-interval`) | `poller.interval` in `config.yaml` (Go duration, 30s to 1h, default 5m) sets the default SNMP poll interval; Settings, General edits it, and the first-install wizard asks for it. A change applies without a restart to the poller (devices replan), reachability, the rollup and the API through one shared value (`store.PollCadence`). A device's own `poll_interval_s` still wins, and `--poll-interval`, when given, still overrides the setting (dev and tests). | Owner request 2026-10-24: the interval is an operator choice (DEVIATIONS 49) and belongs where the other settings are. The one cost is a replan of every device on change. | post-v1.4 |
+| 54 | (not specified) an operator cannot poll one device on demand or see the SNMP exchange | Three admin tools on the device page and API: `poll-now` (wraps the poller's existing kick), `poll-test` (one synchronous, traced cycle of the real poll plan through the poller's own fetch path; no metrics written, no poller state changed; one per device, four overall, 60 s, 5,000 OID rows) and `snmp-debug` (per-device recording of scheduled cycles, at most 60 minutes, last 20 cycles in memory only, cleared on restart, plus one INFO log line per cycle). Credentials never appear in a trace or log. Also `nmon device poll <slug> [--test] [--json]`. | Owner request 2026-10-25: "there's no snmp debug options or an option to individually poll a device for testing purposes." Cost: recordings are lost on restart by design, and a request's duration includes the SNMP library's own retries. | post-v1.4 |
+| 55 | (not specified) a port graph shows no cumulative traffic | `GET /api/v1/graph` adds `totals` (`in_bytes`, `out_bytes`, `covered_seconds`, `range_seconds`): the average-rate series integrated into bytes over the range, per port only. Each tier counts its own step, gaps count nothing, and unreliable ports get none. The UI shows it as "total" in the port summary and a Total column in the full graph, in decimal units. | Owner request 2026-10-25: show bytes transferred per port over the selected timespan, as LibreNMS does. Additive API change. Cost: a total is only as complete as the data, so a partial one is marked. | post-v1.4 |
+| 56 | (not specified) port graphs show bits only, with in/out p95 | LibreNMS parity, additive API only: `p95.aggregate_bps` (95th of in+out per 5-minute slot, both directions present); the UI adds Max and the aggregate to the port summary, utilisation percentages and a "% of speed" axis (from `iface.speed_bps`, hidden when unknown), and a "Compare previous period" overlay (a second `GET /api/v1/graph` over the preceding same-length range, shifted client-side). New `GET /api/v1/graph/counters` serves **errors** and **discards** rates from the hot store **only**: those counters have no rollup (rollup metric names are frozen), so they reach back over hot retention at most and the response and UI say when a range was cut. **Packets and broadcast/multicast graphs are not offered** (shown disabled): the generic profile does not poll the `ifHC*UcastPkts`/`*MulticastPkts`/`*BroadcastPkts` counters, and adding them would add three OIDs per port to every device's poll and change every fixture. Error and discard counters are 32-bit by definition, but the unreliable rule is about octet wrap on fast links and does not apply to them (VictoriaMetrics `rate()` absorbs a counter reset). | Owner request 2026-10-25: "let's match what libre does" for per-port graphs. Cost: the aggregate is live-only (a month older than hot retention has none), a previous period far back falls to hourly rollups and fills only some slots of a finer current axis, and packet-level graphs need a separate polling decision. | post-v1.4 |

docs/dev.md

@@ -11,6 +11,12 @@ nmon serves the UI at `/`. Unknown paths fall back to `index.html` for the clien
 `/api/`, `/docs`, `/openapi*`, `/metrics` and `/schemas`, which belong to the API. Files under `/assets/`
 are content-hashed and sent `immutable`; `index.html` is `no-cache`; text assets are served gzipped.
 
+## In-app help (`/help`)
+
+The operator docs are shown at `/help` and `/help/<doc>#<heading>`. The markdown stays in `docs/` and is imported raw into the web build (`web/src/pages/help/docs.ts`, `?raw` imports), so there is no copy to keep in sync and the text always matches the binary. The page is a lazy chunk (react-markdown + remark-gfm), loaded on first visit. Raw HTML in the markdown is not rendered.
+
+To add a doc: write `docs/<id>.md` (h1 title, h2/h3 sections; the h2/h3 headings become the table of contents and search targets), add one entry to `DOCS` in `docs.ts`, and link to other docs as `docs/<id>.md#heading` (links to files that are not in `DOCS`, such as `docs/spec/`, render as plain text). To give a Settings section a "?", add `section id: heading slug` to `web/src/pages/help/helpLinks.ts`. Heading slugs are GitHub-style (`Behind a reverse proxy` is `behind-a-reverse-proxy`). Developer-only docs (this file, milestones, spec, DEVIATIONS, MAINTAINABILITY) are deliberately not listed. `/help` is an SPA route; `/docs` stays the Swagger UI.
+
 ## OpenAPI compatibility
 
 `scripts/openapi-check.sh` compares the current spec (`nmon openapi`, else `openapi.json`) with

docs/milestones/README.md

@@ -50,6 +50,8 @@ Departures from the spec are recorded in [`docs/DEVIATIONS.md`](../DEVIATIONS.md
 
 - **Web push notifications (v2)**: a `webpush` receiver kind in `internal/notify` (RFC 8291 payload encryption, RFC 8292 VAPID, standard-library crypto), per-user subscriptions (needs M6 users), a service worker in the UI, and push endpoints in the outbound allow-list. Out of v1 per the spec's non-goals ("alerts go to webhooks and email only"). Owner deferred it to v2 on 2026-10-22. Until then, a webhook receiver pointed at ntfy or Gotify gives phone push.
 
+- **Distributed poller (v2)**: remote pollers per site or rule that spool to disk while the master is down, send signed chunks over an outbound-only link (MQTT through a broker), and receive device credentials sealed to their own key. Assessed feasible on 2026-10-25 and deferred to v2 by the owner. Requirements, gaps and risks: [v2-distributed-poller.md](v2-distributed-poller.md).
+
 - **BMC support**: SNMP profiles for Dell iDRAC, HPE iLO and Supermicro (health, temperatures, fans, PSUs, power, RAID) feeding the Health/Power/Environment tabs; a native Redfish collector later. Today, Redfish works only via an exporter pushing with a push token. Owner: "good enough for now" (2026-10-23).
 
 ## Post-v1 additions

docs/milestones/v1.2-custom-alert-rules.md

@@ -0,0 +1,202 @@
+# v1.2: built-in rule macros and custom alert rules
+
+Owner request (2026-10-23), following v1.1 (alert policies): the shipped rules are too constraining. Keep them as
+well-known **built-in macros**, and let admins create their own rules from the data nmon already holds. The owner's
+decisions:
+
+- Custom rules can test **metrics** (anything in the hot store) **and inventory fields** (device and interface rows).
+- **Builder only.** Admins compose rules from whitelisted fields, never by writing MetricsQL or SQL. The generated query
+  is shown read-only.
+
+## Rules after this change
+
+| Kind | Where defined | Editable | Notes |
+|---|---|---|---|
+| Built-in macro | `alert.DefaultRules()` | no; "Copy to custom" clones one into an editable builder rule where the builder can express it, otherwise refuses with a reason | Names are unchanged (`interface_errors`, `disk_full`, …), so v1.1 policies, alert history and series keys stay valid. The UI labels them "Built-in". |
+| Config rule | `config.yaml` `alerting.rules` | file only | It still works but is deprecated, and the startup log says so once. A config rule cannot take a custom rule's name. |
+| Custom metric rule | DB `alert_rules`, `kind: metric` | API/UI, live | A builder spec is compiled to MetricsQL. |
+| Custom inventory rule | DB `alert_rules`, `kind: inventory` | API/UI, live | A builder spec is evaluated in Go over store rows. |
+
+Every rule, of whatever kind, is alerted only through policies (v1.1), and uses the same state machine (pending/`for`,
+firing, resolved, stale, flapping), suppression, maintenance and notifications. A rule name is unique across all kinds,
+and names reserved by reach and system alarms stay refused. Deleting a custom rule that a policy references is a 409
+naming those policies. Disabling a rule retires its open rows silently.
+
+## Builder: metric rules (`kind: metric`)
+
+| Field | Values |
+|---|---|
+| `metric` | from the catalogue: every metric of every loaded profile (name, label, unit, type, category, `of`, enum), the interface metrics (`ifHC*Octets`, `if*Errors`, `if*Discards`, `ifOperStatus`, `ifAdminStatus`, and whatever else the generic profile polls), and push-ingested metric names read from the hot store's `__name__` values (marked `source: push`) |
+| `transform` | `value`, `rate` (counter types only; per second), `percent_of` (only when the metric has `of`), `utilisation` (interface octet counters only: 100 × 8 × rate / speed). Offer `utilisation` only if the port speed is available in the hot store as a series; if it isn't, leave it out and record why in the gate log. Don't invent a join. |
+| `window` | duration, default 5m, minimum 2× the poll interval |
+| `func` | `last`, `avg`, `max`, `min` over the window |
+| `aggregate` | component-level metrics only: `each` (default), or `max`/`min`/`avg`/`sum` per device |
+| `op` | `>`, `>=`, `<`, `<=`, `==`, `!=` |
+| `threshold` | number; for enum/status metrics the UI offers the enum names and stores the number |
+| `for`, `severity`, `summary`, `description` | as on today's rules; the templates are validated and a default summary is generated |
+
+The level (device, interface or component) follows from the metric: `iface` label → interface, index label → component,
+scalar → device. It decides which policy matchers apply, exactly as in v1.1. The MetricsQL is generated from fixed
+templates, and every name and value is checked against the catalogue or a numeric parse, so nothing a user types reaches
+the query text unescaped. Label values are not user input in the builder.
+
+The evaluator's comparison gains `>=`, `<=`, `==` and `!=` (`Rule.breach`; validation accepts all six).
+
+## Builder: inventory rules (`kind: inventory`)
+
+Evaluated every cycle from rows the store already returns (devices and interfaces, which are not retired). Add a store
+method only if no existing one fits; still no SQL outside `internal/store` and no dynamic SQL anywhere. Field whitelist:
+
+- **Device:** `last_seen_age` (duration), `enabled`, `profile`, `profile_pinned`, `site`, `role`, `kind`, `sys_descr`,
+  `hw_serial`, `mgmt_ip`, `parent_count`, `snmp_failing` (if it is cheaply derivable, otherwise leave it out).
+- **Interface:** `if_speed_mbps`, `admin_status`, `oper_status`, `if_name`, `if_alias`, `description`, `unreliable`,
+  `has_hc_counters`, `excluded`, `counter_discontinuity`, `last_discovered_age`.
+
+Ops by type:
+
+| Type | Ops |
+|---|---|
+| number | `> >= < <= == !=` |
+| bool | `is true`, `is false` |
+| string | `equals`, `not equals`, `matches` (RE2, compiled at save), `empty`, `not empty` |
+| duration | `older than`, `newer than` |
+
+A rule has one to five conditions, ANDed, all on device fields or all on interface fields; that decides the level. The
+reported value is 1 when the conditions hold, and an entity whose conditions don't hold is "below". Examples:
+
+- "Uplink without description": interface, `if_alias empty` AND `if_speed_mbps >= 10000`.
+- "Device not seen for 1 h": device, `last_seen_age older than 1h`.
+
+## Architecture
+
+- New pure package `internal/alertrule`:
+  - spec types and `Validate`;
+  - `CompileMetric(spec, catalogue) (expr, level, error)`;
+  - `EvalInventory(spec, row) bool` over a plain row struct;
+  - the field whitelist and the catalogue types.
+  - Unit tests carry a table of specs → expected MetricsQL, and of rows → verdicts.
+- `internal/alert`:
+  - The evaluator reads its rule set from a `Rules func() []Rule` (or a `SetRules`) so DB edits apply on the next cycle
+    without a restart.
+  - A rule gets a `Source`: `metric` (today's querier path) or `inventory`, an interface the app supplies that returns
+    readings for a rule at `now`.
+  - The state machine is shared and unchanged.
+- `internal/store`:
+  - migration `20261023000013_alert_rules.go` (frozen struct; spec as JSON text);
+  - `AlertRules` sub-interface in a new file, plus one embed line;
+  - the table is added to `backupTables`.
+- `internal/app`:
+  - the rule registry: built-ins, then config, then DB;
+  - a 30 s cache invalidated on write;
+  - the metric catalogue from the profile set plus the hot store's metric names (cached);
+  - the inventory source, built on TTL-cached store reads.
+- `internal/api`, all additive. Admin unless noted; regenerate `openapi.json` and the web schema and run
+  `scripts/openapi-check.sh`.
+  - `GET /api/v1/alert-rules` keeps its fields and adds `id`, `kind` (`builtin`, `config`, `metric`, `inventory`),
+    `editable`, `spec` and `expr`.
+  - `GET /api/v1/alert-rules/{id}`, plus `POST`, `PUT` and `DELETE` for custom rules only (a built-in is 409).
+  - `GET /api/v1/alert-rules/catalogue` (viewer) returns the metrics and the inventory fields with their types and ops.
+  - `POST /api/v1/alert-rules/test` (viewer; declare it read-only in the auth policy) compiles a draft and runs it once.
+    It returns the generated expr, the series or entity count, the number breaching now, and up to 20 samples
+    (device, iface, component, value).
+  - `POST /api/v1/alert-rules/{id}/copy` clones a built-in into a custom rule, or returns 422 with the reason when the
+    builder can't express it.
+- `web/`:
+  - An **Alert rules** tab next to Alert policies. The list shows the kind badge, level, condition text, policy count and
+    enabled state.
+  - A builder dialog:
+    - pick metric or inventory;
+    - searchable catalogue picker grouped by category;
+    - dependent fields (transform shown only where valid, enum picker for status metrics);
+    - generated query shown read-only;
+    - a **Test** button showing "N breaching now" with samples;
+    - **Copy to custom** on built-ins.
+  - The policy dialog's rule dropdown lists custom rules too.
+  - Mock handlers and an e2e spec. Keep web files ≤ 400 lines; split the builder.
+
+## Docs
+
+- DEVIATIONS: a row noting that rules are now also defined in the database, with a builder only and inventory
+  conditions (owner decision 2026-10-23), and that config rules are deprecated.
+- `docs/operations.md`: extend the alert section with the two inventory examples above and a metric example ("CPU over
+  80 % on role=server for 15 min"), and with how a custom rule and a policy combine.
+
+## Gate
+
+- `make lint` is clean. All of these pass:
+  - `go test -race ./...`
+  - `go test -race -tags 'integration vmembed'` for `internal/alert`, `internal/app` and `internal/store`, where every
+    builder template and every built-in is run against a real VictoriaMetrics.
+  - `./scripts/test-pg.sh` and `scripts/openapi-check.sh`
+  - web lint, test and build, plus the new and changed e2e specs only. A full Playwright run isn't needed; revert any
+    screenshots it rewrites.
+- Tests prove the following:
+  - A custom metric rule, bound by a policy, fires and resolves.
+  - A custom inventory rule fires for a matching port and not for others.
+  - A rule edit applies on the next cycle without a restart.
+  - Deleting a referenced rule is a 409.
+  - Built-ins cannot be edited.
+  - A builder spec with an unknown metric or field is a 422.
+  - Hostile input in a string field (quotes, braces, `"} or vector(1)`) never changes the generated query's structure.
+
+## Gate log
+
+2026-10-24, run by the implementing agent:
+
+- Decisions where this spec was silent:
+  - **`utilisation` is not offered.** No profile polls `ifHighSpeed` or `ifSpeed` as a series (checked with a grep of
+    `profiles/*.yaml`); port speed lives only in the inventory table. Joining the two would be inventing a join, so the
+    transforms are `value`, `rate` and `percent_of`. A test (`TestCatalogueHasProfileAndPushMetrics`) fails if
+    `ifHighSpeed` ever becomes a series, as a prompt to revisit this.
+  - **Counters allow `rate` only** (a raw counter is never alertable), and `rate` always averages over the window
+    (`func` must be `avg` or empty). The default window is the larger of 5 m and twice the metric's own poll interval
+    (so a 5 m SNMP metric defaults to 10 m); an explicit window below twice the interval, below 1 m or above 24 h is a
+    422. A `rate` of a `counter32` interface metric sets `SkipUnreliable`.
+  - **Inventory fields kept:** `last_seen_age`, `profile`, `profile_pinned`, `site`, `role`, `kind`, `sys_descr`,
+    `hw_serial`, `mgmt_ip`, `parent_count`, `snmp_failing` (true while the poller reports `degraded` or `backoff`),
+    `if_speed_mbps` (ifHighSpeed, else ifSpeed / 10^6), `admin_status`, `oper_status` (numbers, with enum names for
+    the UI), `if_name`, `if_alias`, `description`, `unreliable`, `has_hc_counters`, `excluded`, `counter_discontinuity`,
+    `last_discovered_age`. **Dropped: `enabled`.** Disabled, retired and purged devices are never evaluated (their
+    alerts close as retired through `Active`), so `enabled is false` could never match. A device never seen has its age
+    counted from `created_at`. Excluded ports are evaluated (so `excluded` is a usable condition). A new store method,
+    `ListAllInterfaces` (one query, sub-interface `InterfaceInventory`), avoids one query per device.
+  - **Built-ins that can be copied to custom:** `cpu_high` (compiles to the identical expression),
+    `ups_on_battery` (`upsOutputSource == 5`) and `printer_supply_empty` (`prtMarkerSuppliesLevel == 0`). The other
+    seven (and the reachability and system rules) join, filter or union series, so `POST /alert-rules/{id}/copy` answers
+    422 with the reason. The copy is created at once as `<name>_custom` (or the name in the body), keeping the
+    built-in's severity, hold time and texts.
+  - Rules are classed `builtin` when the name and expression match the shipped rule (a config patch of thresholds
+    keeps it `builtin`), and `config` otherwise. A custom rule that cannot compile at runtime (a pushed metric briefly
+    missing from the hot store) keeps its last good form instead of retiring its alerts. The rule list grows `policy_count`
+    and `copyable`; deleting a rule, renaming one, or changing its level while a policy names it is a 409. Editing
+    `GET /alert-rules/{id}` is viewer-level like the list; `POST /alert-rules/test` is the only read-only POST.
+  - Type names in the OpenAPI schema are `RuleSpec`, `InventoryField` and `InventoryCondition` (not the bare words).
+- `make lint`: 0 Go issues, filesize script clean, ESLint `--max-warnings 0` clean (see the final run below).
+- `go test -race ./...`: all 44 packages ok (the new `internal/alertrule` included). An earlier run found a real
+  deadlock in the first version of rule update (a second store call inside a SQLite transaction); fixed by reading
+  the old level before the transaction, covered by `TestDeletingOrReshapingAReferencedRuleIs409`.
+- `go test -race -tags 'integration vmembed' ./internal/alert/... ./internal/app/... ./internal/store/...`: ok. The new
+  `TestIntegrationBuilderTemplatesAgainstRealStore` compiles every allowed transform x function x aggregate for
+  gauge, status, counter32, counter64, percent_of and push metrics and runs each query on a real VictoriaMetrics; the
+  existing test still runs every built-in there.
+- `./scripts/test-pg.sh` (store suite on Postgres 16, includes migration 13 and the rule and bulk-interface queries): ok.
+- `scripts/openapi-check.sh`: compatible with m6 (additive; the `op` enum of `AlertRuleInfo` only gained values).
+  `openapi.json` and `web/src/api/schema.d.ts` regenerated.
+- Web: `npm run lint`, `npm run test` (146 tests) and `npm run build` pass. Playwright, only the added or changed specs
+  (`E2E_PORT=4795`): `alert-rules.spec.ts` (7 tests) and `alert-policies.spec.ts` (4, its mock now shares the rule store)
+  pass. The full Playwright suite was not run; `docs/screenshots` was reverted and no preview server was left running.
+- The tests that prove the gate items:
+  - Custom metric rule bound by a policy fires and resolves, and a rule edit (new threshold) applies on the next
+    cycle with no restart, and disabling retires silently:
+    `internal/app` `TestCustomMetricRuleBoundByAPolicyFiresResolvesAndEditsApplyLive`; the evaluator side,
+    `internal/alert` `TestRuleSetChangesApplyOnTheNextCycle`.
+  - Inventory rule fires for the matching port only: `internal/app` `TestInventoryRuleFiresForTheMatchingPortOnly`.
+  - Deleting a referenced rule is 409 (naming the policies): `internal/api` `TestDeletingOrReshapingAReferencedRuleIs409`.
+  - Built-ins cannot be edited or deleted, and their names cannot be taken: `TestBuiltinRulesCannotBeEditedOrDeleted`,
+    `TestCustomRuleNamesCannotShadowBuiltinsOrConfig`.
+  - An unknown metric or field is a 422: `TestBuilderSpecWithUnknownMetricOrFieldIs422`, `alertrule` `TestCompileMetricRefusals`,
+    `TestInventoryRefusals`.
+  - Hostile input never changes the generated query: `alertrule` `TestHostileTextNeverReachesTheQuery` (quotes, braces,
+    `"} or vector(1)`, backticks and template braces in name, summary and description leave the expression identical or
+    are refused; a pushed metric name that is not a plain identifier never enters the catalogue); the `window`
+    is parsed and re-printed, and `TestCompileMetricTemplates` pins the exact MetricsQL of every template.

docs/milestones/v1.3-bgp-and-l2-tables.md

@@ -221,9 +221,9 @@ Not run: the `make gate-m*` scripts, the Ansible tasks (no Ansible here; the YAM
 ### Decisions where the spec was silent
 
 - **L2 settings and reload.** `l2.*` and `oui.*` apply from the next cycle after a reload (SIGHUP or a Settings save) with no restart, and are reported as `applied`. The job re-reads `l2.interval` every minute while waiting, so a shorter interval takes effect within a minute and a longer one on the wait in progress. Test: `TestRunStartsACycleAtOnceThenEveryIntervalAndReadsTheIntervalAgain`.
-- **Ranking.** Edge before uplink, then rows the last walk still sees before history, then fewest MACs, then most recent (DEVIATIONS 55).
+- **Ranking.** Edge before uplink, then rows the last walk still sees before history, then fewest MACs, then most recent (DEVIATIONS 48).
 - **Per-VLAN fallback.** If the first VLAN context does not answer, the walker falls back to the plain tables instead of waiting out a timeout per VLAN.
-- **`bgp_peer_down` text.** AS and description are added to the notification, not the stored alert row (DEVIATIONS 53).
+- **`bgp_peer_down` text.** AS and description are added to the notification, not the stored alert row (DEVIATIONS 46).
 - **BGP per profile.** See `docs/coverage.md`. Nothing for mikrotik-routeros (no BGP objects in MIKROTIK-MIB or the recordings) and the non-router profiles; cisco-nxos is unverified against a recording; VyOS is covered through BGP4-MIB on the strength of the FRR recording only.
 - **Open item for v1.2.** The prefix-drop example in `docs/operations.md` is to be added once custom rules land.
 

docs/milestones/v2-distributed-poller.md

@@ -0,0 +1,72 @@
+# v2: distributed poller (feasibility, deferred)
+
+Status: deferred to v2 by the owner on 2026-10-25. This is a feasibility assessment, not a plan; nothing is built.
+
+## Requirements (owner)
+
+1. A remote poller polls the devices of a site, or of a predefined rule.
+2. It sends its data to the master only as chunked payloads.
+3. While the master is down it keeps polling, and delivers when the master is back.
+4. Each poller has its own private key; the master sends the target device credentials sealed to it.
+5. It works over an outbound-only link (MQTT pub/sub through a broker). Neither side accepts a connection from
+   the other. Not a data diode yet, but the design should not rule one out.
+
+## Verdict
+
+Feasible, about one milestone. The poller is already injectable: `poller.Options` (`internal/poller/poller.go`)
+takes the store, the metric writer, the reachability observer and the SNMP dialer, so the polling code can be
+reused unchanged. The work is in what sits around it: where targets come from, where samples go, and how
+alerting treats data that arrives late.
+
+## What exists and what is missing
+
+| # | Exists | Missing |
+|---|---|---|
+| 1 | `devices.site`, `store.DeviceFilter.Site`; the alert-policy matcher (`alertpolicy.Device{Slug,Profile,Role,Site,Kind}`) already selects devices by rule | a `pollers` table and assignment rules; reuse the policy matcher rather than a second rule engine |
+| 2 | `metrics.Sample`; the vmsup queue batches 50k samples per POST | a chunk format (poller id, sequence, signature). Push ingest is **not** reusable: it strips `device`/`iface`, refuses `if*`/`nmon_` names and rejects timestamps more than 1 h from now. Poller data needs its own trusted path into the hot queue |
+| 3 | the hot queue buffers in memory only (60 s / 2M samples, then drops) | a disk spool on the poller (append-only segments, bounded by size), and a local SQLite snapshot of its targets so it restarts without the master |
+| 4 | symmetric AES-256-GCM keyring with versions (`internal/secrets`, `store/cred.go` `seal`); rotation in `internal/keyrotate` | a key pair per poller (X25519 + Ed25519): the master seals the credential bundle to the poller's public key (HPKE) and signs it; the poller signs its chunks; rotation follows the keyrotate pattern |
+| 5 | none | an MQTT transport behind an interface (see below), with outbound HTTPS chunk upload as a second, brokerless option |
+
+## Transport (requirement 5)
+
+- Data up: `nmon/poller/<id>/data`, QoS 1. The master holds a persistent session, so the broker keeps chunks
+  while the master is down.
+- Config and credentials down: a retained, sealed message on `nmon/poller/<id>/config`, read on every connect.
+- An MQTT client library (e.g. paho.golang) departs from the spec's "standard library elsewhere" and needs a
+  DEVIATIONS row.
+
+Rules that keep a later data diode possible:
+
+1. The data path is already one-way. Chunks are self-describing, signed and sequence-numbered, and the master
+   drops duplicates by (poller, seq), so a resend is harmless. The only signal the poller waits for is the
+   broker's PUBACK, which frees spool space. Under a diode, spool release becomes age/size bound plus a periodic
+   resend window; the format does not change.
+2. One downlink only: the sealed, signed config bundle. Make it a file format that arrives either as the
+   retained message or by hand (`nmon poller import bundle.sealed`). A diode drops the subscription and keeps
+   the import.
+3. Interactive tools (poll-now, test poll, SNMP debug recording: `PollTools`) need a request and a reply. Over
+   the broker they work through a request topic with seconds of latency. Under a diode they are unavailable,
+   and the UI must say so for remote devices.
+
+## Risks
+
+1. **Late data looks like an outage.** `reach.Evaluator` runs on the master and expects live
+   `ObserveICMP`/`ObserveSNMP` (`internal/reach/types.go`); the API marks a device down after three silent
+   intervals. A site link drop would page every device behind it. Fix: model the poller as a dependency parent
+   of its devices (dependency suppression already exists) and replay spooled probes as history, never as live
+   state transitions. Needs a DEVIATIONS row.
+2. **Inventory writes from the poller.** The poller writes poll state (`devloop.persistState` →
+   `SetPollState`), component rows and discontinuity bases. Remotely these go to the local snapshot and are
+   sent up as events.
+3. **Discovery must run remotely.** The master usually cannot reach a site's devices. Interface ULIDs are the
+   frozen `iface` label; the poller can mint them (ULIDs are globally unique) and the master accepts them.
+   Traps, L2 tables and checks need the same scope decision up front.
+4. **Late writes.** VictoriaMetrics accepts old timestamps within retention; dedup by (poller, seq) keeps a
+   resent chunk from being written twice.
+
+## Codebase constraints
+
+- A new package (e.g. `internal/remotepoll`) and an `nmon poller` subcommand wired in `internal/app`.
+- Collection paths still do not import each other; time comes from `internal/clock`.
+- API routes additive only (`scripts/openapi-check.sh`); new tables through a new migration.

docs/operations.md

@@ -2,14 +2,93 @@
 
 How to run, secure, back up and maintain a production nmon. Design background: `docs/spec/spec.md` ("Auth", "Operations").
 
+This page, [Adding a device profile](docs/profiles.md) and [Collection coverage](docs/coverage.md) are also built into the binary and shown in the UI under **Help** (`/help`, any signed-in user, no network needed). The help page always matches the running version; the Settings forms link their section's "?" to the matching heading here.
+
 ## Starting
 
     nmon serve --data-dir /var/lib/nmon --secret-key-file /etc/nmon/secret.key \
                --tls-cert /etc/nmon/tls.crt --tls-key /etc/nmon/tls.key
 
-- **HTTPS is the default.** Pass a certificate pair (`--tls-cert`, `--tls-key`; `SIGHUP` reloads them without dropping connections) or Let's Encrypt (`--acme-host nmon.example.net` repeatable, `--acme-email you@example.net`; nmon answers the http-01 challenge on `--acme-listen`, default `:80`, certificates are cached in `<data-dir>/acme`). Without either, `serve` refuses to start and says so. `--insecure-http` is for development: plain HTTP, cookies without `Secure`.
-- **A sign-in method is required** over TLS (`auth.oidc` and/or `auth.break_glass_file` in `config.yaml`). With `--insecure-http` and neither configured nmon runs in *dev-open mode* (everyone is admin `dev`; a banner says so); never expose that port.
-- Behind a reverse proxy, list it in `auth.trusted_proxies`; `X-Forwarded-*` is ignored from anyone else.
+- **HTTPS is the default.** Pass a certificate pair (`--tls-cert`, `--tls-key`; `SIGHUP` reloads them without dropping connections) or Let's Encrypt (`--acme-host nmon.example.net` repeatable, `--acme-email you@example.net`; nmon answers the http-01 challenge on `--acme-listen`, default `:80`, certificates are cached in `<data-dir>/acme`). With neither (and no `--http-listen`) nmon generates a self-signed certificate, see [The self-signed certificate](#the-self-signed-certificate). `--insecure-http` is for development: plain HTTP, cookies without `Secure`.
+- **A sign-in method is required** over TLS (`auth.oidc` and/or `auth.break_glass_file` in `config.yaml`). On a first start with none configured nmon opens the [setup wizard](#installing-nmon-first-start) instead of refusing. With `--insecure-http` and neither configured nmon runs in *dev-open mode* (everyone is admin `dev`; a banner says so); never expose that port.
+- Behind a reverse proxy, see [Behind a reverse proxy](#behind-a-reverse-proxy): `--http-listen ADDR` adds an opt-in plain-HTTP listener, and the proxy goes in `auth.trusted_proxies`; `X-Forwarded-*` is ignored from anyone else.
+
+## Installing nmon / first start
+
+On a fresh install nmon needs no `config.yaml`: start it, open `https://<host>:8443/` and the browser lands on `/setup`, a five-step wizard (sign-in, basics, first devices, alerting, finish). Steps 2 to 4 can be skipped and done later in Settings.
+
+    nmon serve --data-dir /var/lib/nmon --secret-key-file /etc/nmon/secret.key
+
+### Install mode
+
+Install mode is on only while **no sign-in method is configured** (no `auth.oidc`, no `auth.break_glass_file`) **and setup has never been completed**. While it is on:
+
+- **Anyone who can reach nmon can run setup**, as an admin. There is no setup token and no localhost restriction: nmon is meant to sit in a DMZ or an internal network (`docs/DEVIATIONS.md` row 51). Finish setup right after starting, or keep the port firewalled until you have.
+- nmon logs a `WARN` at start and every 10 minutes: `INSTALL MODE: setup is open to anyone who can reach this address`.
+- Only the wizard works. The UI, `GET /api/v1/setup`, the settings form, webhook presets/preview/test, adding devices (and CSV import), alert policies, the profile list, secrets and `GET /api/v1/meta` are served; **every other API route answers `503` "setup not finished"** until setup completes.
+
+Install mode ends **for good** when setup completes (`POST /api/v1/setup/complete`, the wizard's last step). The completion is stored in the inventory database and nothing on the web can undo it. If you later delete the break-glass file or the `auth:` block, nmon does not reopen setup: it refuses to start with "no way to sign in", as before. An existing install that already has a sign-in method is marked completed the first time it starts on this version, so an upgrade can never open install mode. `--insecure-http` (development) never enters install mode.
+
+To skip install mode altogether, configure sign-in before the first start, as the Ansible role does (`nmon_break_glass_file` / `nmon_oidc_*`): such an install is treated as already set up.
+
+### Setup wizard: sign-in
+
+Required. Pick the break-glass administrator (default `admin`) and a password of at least 12 characters. nmon hashes it with bcrypt, writes `<data-dir>/breakglass` (mode 0600, `user:hash`) and sets `auth.break_glass_file`. Optionally add an OpenID Connect provider: issuer, client id, redirect URL (prefilled from the address you are using plus `/auth/callback`), groups claim, admin group and the client secret, which is stored encrypted as the secret `oidc-client`. nmon checks the fields now and contacts the provider at the first sign-in. Sign-in to the web UI has no login page: OIDC redirects the browser to the provider, and the break-glass admin signs in at `/auth/breakglass` (HTTP Basic).
+
+### Setup wizard: basics
+
+The time zone (site default for p95 months), retention (raw samples `retention.hot` 30d, five-minute rates 120d, hourly rates 2y) and the default poll interval (`poller.interval`, 30s to 1h, default 5m). All of them are also in Settings, General and Retention.
+
+### Setup wizard: first devices
+
+Choose an SNMP credential (v2c community, or v3 user with auth and privacy), then paste IP addresses (one per line, IPv6 or IPv4) or upload a CSV. Each address is added like the Add device dialog does it, so a device that does not answer SNMP is reported and not added. Host names are not resolved; use the address. The step shows each device's discovery state live.
+
+### Setup wizard: alerting
+
+Add one receiver, a webhook (with the Teams, Slack, Discord and generic JSON presets, a live preview and Send test) or email, and optionally turn on the recommended alert policies: `device_down`, `device_warning`, `interface_errors`, `cpu_high`, `memory_high` and `disk_full`, each for all devices. Receivers get every notification, so the policies need no binding to the new receiver. Running the step again creates nothing twice. Interface-down alerts stay opt-in (see below).
+
+### Setup wizard: finish
+
+The wizard summarises what it set up and calls `POST /api/v1/setup/complete`. nmon checks that a sign-in method now exists (otherwise `409`), stores the completed flag, and switches authentication on **without a restart**: the live auth service is replaced by one built from the new configuration, and install mode closes. The browser is then sent to sign in.
+
+### The self-signed certificate
+
+When none of `--tls-cert`/`--tls-key`, `--acme-host` or `--http-listen` is given, nmon serves HTTPS with a self-signed ECDSA P-256 certificate kept in `<data-dir>/tls/` (`cert.pem`, `key.pem` mode 0600). It is created once and reused on every start; it is valid for about two years and replaced within 30 days of expiry. Its names are the host name, `localhost`, `::1`, `127.0.0.1` and every non-loopback interface address (IPv6 included), as they were when it was created: delete the two files to make a new one after the host's addresses change. Browsers warn about it, and nmon logs a `WARN` at each start. Replace it with `--tls-cert`/`--tls-key` (reloaded on `SIGHUP`) or `--acme-host`. With `--http-listen` alone nmon serves plain HTTP only and creates no certificate.
+
+## Behind a reverse proxy
+
+    nmon serve --data-dir /var/lib/nmon --secret-key-file /etc/nmon/secret.key --http-listen [::1]:8080
+
+`--http-listen ADDR` (default off; any interface is allowed, but loopback or an internal address is the point) adds a **plain-HTTP listener** that serves the same UI and API. HTTPS stays the default and is unchanged.
+
+- Beside HTTPS: with `--tls-cert/--tls-key` or `--acme-host` too, nmon serves HTTPS on `--listen` and plain HTTP on `--http-listen`, so the proxy may reach it either way. Alone: with no TLS configured nmon starts with only the HTTP listener (`--listen` is not bound) instead of refusing. SIGHUP still reloads the certificate for the HTTPS listener.
+- **Sign-in stays on** over this listener (OIDC, break-glass, API tokens), exactly as on HTTPS. It is not dev-open mode: with no `auth.oidc`/`auth.break_glass_file` nmon refuses to start. `--http-listen` and `--insecure-http` are mutually exclusive (`--insecure-http` is development mode with no sign-in).
+- List the proxy in `auth.trusted_proxies` (addresses or CIDRs, live-reloadable). Only a request from a trusted proxy has its headers honoured: `X-Forwarded-For` gives the client address used by the audit log and the break-glass lockout (the rightmost hop that is not itself a trusted proxy), and `X-Forwarded-Proto: https` marks the browser connection as HTTPS. From anyone else those headers are ignored and removed, so they cannot be spoofed. With the list empty nmon logs a WARN at startup: every client is seen as the proxy's address, so the lockout is per-proxy.
+- **Cookies:** the session and sign-in-flow cookies are always `Secure` on the HTTPS listener. On the HTTP listener they are `Secure` only when a trusted proxy sent `X-Forwarded-Proto: https`; a direct plain-HTTP client gets them without `Secure` (a browser would drop them otherwise).
+- **OIDC:** `auth.oidc.redirect_url` is configured explicitly and is never derived from the request, so set it to the public URL (`https://nmon.example.net/auth/callback`). CSRF uses a per-session token, not the Origin header, so a different public host or scheme needs nothing else.
+- nmon uses no websockets or long-lived streams; the proxy needs no upgrade headers. Raise the proxy's body-size limit if you import large CSV files.
+
+nginx, proxying to the plain-HTTP listener:
+
+    server {
+        listen 443 ssl;
+        listen [::]:443 ssl;
+        server_name nmon.example.net;
+        ssl_certificate     /etc/ssl/nmon.example.net.crt;
+        ssl_certificate_key /etc/ssl/nmon.example.net.key;
+        client_max_body_size 64m;
+
+        location / {
+            proxy_pass http://[::1]:8080;
+            proxy_set_header Host              $host;
+            proxy_set_header X-Real-IP         $remote_addr;
+            proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_read_timeout 120s;
+        }
+    }
+
+with `auth: {trusted_proxies: ["::1"]}` in `config.yaml`. To talk TLS to nmon instead, run nmon with its normal `--tls-cert/--tls-key` (no `--http-listen` needed), use `proxy_pass https://[::1]:8443;` with `proxy_ssl_verify off;` (or `proxy_ssl_trusted_certificate` for a CA-signed pair), and the same `proxy_set_header` lines; the listener is HTTPS, so cookies are `Secure`. Ansible: set `nmon_http_listen: "[::1]:8080"` (and `nmon_tls_mode: none` to serve only that listener).
 
 ## Sign-in
 
@@ -53,7 +132,7 @@ It verifies inventory row counts and a sample of series before it says done. The
 
 ## Configuration
 
-Edit in Settings (admin) or `config.yaml` + `SIGHUP`. An invalid document is rejected and the running configuration stays. Alert receivers, the outbound allow-list, trusted proxies, the backup schedule, disk thresholds and the purge period apply live; everything else is listed as `restart_required` until the next restart. `disk: {alert_used: 0.8, admit_used: 0.9, reserve_free: 0.05}`: alert at 80% used, no new series at 90%, ingest throttled / rollup paused / backups refused below 5% free.
+Settings (admin) is point-and-click: a section list (General, Retention, Sites, Notifications, Authentication, Backup, Disk guard, L2 and OUI, Proxmox, Checks, Alerting) with a form per section, its own Save and Revert, and the applied / restart-required result under it. Unset fields show the default as a placeholder; durations take `30d`, `12h`, `5m`. A form that refers to a secret (OIDC client secret, Proxmox token) takes the secret's name and has a "Set secret value" button that stores the value write-only through `PUT /api/v1/secrets/{name}`; it is never shown again. The same sections are available as typed JSON: `GET /api/v1/settings/form` and `PUT /api/v1/settings/form` (each section present in the body replaces that section; the others are untouched). Alert rules and policies are not here, they are under Alerts; `alerting.rules` in `config.yaml` is preserved by the forms. The Advanced (YAML) tab and `PUT /api/v1/settings` still edit the whole document, and `config.yaml` + `SIGHUP` still works; all three go through the same validation. Saving rewrites `config.yaml` from the parsed configuration, so comments in the file are not kept. An invalid document is rejected and the running configuration stays. Alert receivers, the outbound allow-list, trusted proxies, the backup schedule, disk thresholds and the purge period apply live; everything else is listed as `restart_required` until the next restart. `disk: {alert_used: 0.8, admit_used: 0.9, reserve_free: 0.05}`: alert at 80% used, no new series at 90%, ingest throttled / rollup paused / backups refused below 5% free.
 
 ## Metric stores
 
@@ -80,6 +159,43 @@ Fiber and uplinks alert on any error, WiFi never alerts on errors:
  "scope": {"interfaces": {"if_types": [71]}}}
 ```
 
+## Alert rules: built-in and custom
+
+The shipped rules (`interface_down`, `interface_errors`, `cpu_high`, `memory_high`, `disk_full`, `ups_on_battery`, `ups_runtime_low`, `printer_supply_low`, `printer_supply_empty`, `temp_high`) are read-only **built-in** rules, listed with their generated query under Alerts > Alert rules. `cpu_high`, `ups_on_battery` and `printer_supply_empty` have **Copy to custom**, which clones them into an editable builder rule; the others join, filter or union series and say why they cannot be copied. Rules in `config.yaml` (`alerting.rules`) still work but are deprecated, and the startup log says so; define rules in the UI instead. A rule name is unique across all kinds, and `device_down`, `device_warning`, `system` and the subsystem alarm names stay reserved.
+
+Admins add their own rules with the builder (UI or `POST /api/v1/alert-rules`). There is no free-form query: nmon builds it from whitelisted parts, and the generated MetricsQL is shown read-only. **Test** runs the draft once and shows how many entities breach now. Edits apply on the next cycle; disabling a rule or deleting it retires its open alerts silently, and deleting a rule that a policy still names is refused (409, naming the policies).
+
+- Metric rule, "CPU over 80 % on role=server for 15 min": metric `hrProcessorLoad`, aggregate `max` (one value per device), op `>`, threshold `80`, hold time `15m`; then a policy for this rule with `devices.roles: [server]`. Transforms are `value`, `rate` (counters, per second) and `percent_of` (metrics with an `of`, such as `hrStorageUsed`); the window defaults to 5 min (at least twice the poll interval). Push-ingested metrics are device-level.
+- Inventory rule, "Uplink without description": interface fields `if_alias empty` AND `if_speed_mbps >= 10000`.
+- Inventory rule, "Device not seen for 1 h": device field `last_seen_age older_than 1h`.
+
+A custom rule, like a built-in, alerts nowhere until an alert policy binds it: the rule defines **what** is measured and its default threshold and hold time, and the policy defines **where** it applies and may override severity, threshold and `for`. Several policies can bind one rule to different scopes.
+
+## Webhook templates
+
+Each webhook receiver (Settings > Notifications) has a Go `text/template` that renders the request body. It must produce one valid JSON document (max 256 KB); it is sent as `application/json`. Empty means the default, `{{json .}}`, the whole notification. SMTP receivers use the same editor for `Subject: ...`, a blank line and the body.
+
+**Fields** (a template sees nothing else): `.IncidentID`, `.AlertID`, `.Rule`, `.State` (`firing`, `resolved`, `flapping`, `stale`, `suppressed_summary`), `.Severity` (`critical`, `warning`, `info`), `.Device.Slug`, `.Device.Name`, `.Interface.ID`, `.Interface.Name`, `.Interface.Alias` (empty for device-level alerts), `.Summary`, `.Description`, `.FiredAt`, `.ClearedAt` (nil until resolved), `.Labels`, `.Parents`, `.SuppressedChildren`, `.URL` (link to the device in nmon).
+
+**Functions:** `json`, `upper`, `lower`, `join`, `timefmt` (`{{timefmt .FiredAt "2006-01-02 15:04:05 MST"}}`, UTC), plus the standard `printf`, `eq`, `or`, `if`.
+
+**The `{{json}}` rule:** wrap every value you put inside a JSON string or value in `{{json ...}}` (`"text": {{json .Summary}}`, never `"text": "{{.Summary}}"`). `json` adds the quotes and escapes quotes, newlines and control characters, so an alert text can never break the document. Secret-looking keys and `key=value` pairs are redacted from the rendered payload before it is stored or sent.
+
+**Presets:** "Start from preset" in the receiver form fills the template (it asks before replacing a non-empty one). Built in, also at `GET /api/v1/notify/presets`:
+
+| Preset | For |
+|---|---|
+| Generic JSON | the default; the whole notification |
+| Microsoft Teams (Workflows) | Adaptive Card 1.4 for the Workflows trigger "When a Teams webhook request is received" |
+| Slack | incoming webhook: text fallback and a coloured attachment with blocks |
+| Discord | webhook with one embed, coloured by state and severity |
+
+Teams: Microsoft is retiring the Office 365 connectors ("Incoming Webhook" in a channel's Connectors). Create a Workflow from the template "Post to a channel when a webhook request is received" and paste its URL. The preset is an Adaptive Card message and will not work with a legacy connector URL.
+
+**Preview and test:** the editor shows the server-rendered payload for a sample notification in any state as you type, or the template or JSON error. "Send test" delivers one sample immediately (no retry queue; nothing stored except an `notify.test` audit row) and shows the HTTP status, a response excerpt (4 KB, redacted), the duration and any error. Blank header values and a blank SMTP password use the stored receiver of the same name, so you can test a saved receiver without retyping credentials. Admin only; at most two tests run at once. `POST /api/v1/notify/preview` and `POST /api/v1/notify/test` take `{"receiver": {...form shape...}, "state": "firing"}`.
+
+**Outbound allow-list:** test sends and real deliveries go through the same gate. A webhook to a private, loopback or link-local address is refused unless it matches `alerting.outbound_allow` (Settings > Notifications), and the test result says so.
+
 ## Retired devices
 
 `POST /api/v1/devices/{slug}/purge` (admin) deletes a retired device's series from both stores once it has been retired for `retired_purge_after` (default 1y).
@@ -104,7 +220,7 @@ oui:
   refresh: 7d
 ```
 
-All of these apply from the next cycle after a reload (Settings save or SIGHUP) with no restart. The job never runs on the 60 s poll path and yields to the poller the way the rollup job does. A device that does not answer in time is skipped for that cycle and named on the Poller page (`l2` status and an `l2_devices_skipped` note); it never delays the other devices. A Cisco IOS switch keeps one bridge table per VLAN: the `cisco-ios` profile sets `l2.per_vlan_context: true`, which lists the VLANs from CISCO-VTP-MIB and walks each with community `<community>@<vlan>` (v3: context `vlan-<vlan>`). Other devices use Q-BRIDGE-MIB, then BRIDGE-MIB (VLAN unknown).
+All of these apply from the next cycle after a reload (Settings save or SIGHUP) with no restart. The job never runs on the poll path and yields to the poller the way the rollup job does. A device that does not answer in time is skipped for that cycle and named on the Poller page (`l2` status and an `l2_devices_skipped` note); it never delays the other devices. A Cisco IOS switch keeps one bridge table per VLAN: the `cisco-ios` profile sets `l2.per_vlan_context: true`, which lists the VLANs from CISCO-VTP-MIB and walks each with community `<community>@<vlan>` (v3: context `vlan-<vlan>`). Other devices use Q-BRIDGE-MIB, then BRIDGE-MIB (VLAN unknown).
 
 API: `GET /api/v1/l2/search?q=`, `GET /api/v1/devices/{slug}/{l2,fdb,arp,l2/ports}`, `GET /api/v1/interfaces/{id}/macs`. On a device, the **L2** tab (MAC table with a VLAN filter, ARP / ND, VLAN list) appears once the job has read it, and the Ports tab shows each port's MAC count linking to that port's MACs.
 
@@ -116,6 +232,30 @@ The registry is the IEEE's public data: MA-L (`https://standards-oui.ieee.org/ou
 
 ## BGP
 
-Router profiles collect BGP session state, uptime, accepted prefixes per address family and update rates every 5 minutes (see `docs/coverage.md` for which profiles and MIBs). The **BGP** page lists every session on every device, the ones that are down first, filterable by state, AS, family and text; a device's **Routing** tab shows its sessions with click-to-graph on prefixes and updates.
+Router profiles collect BGP session state, uptime, accepted prefixes per address family and update rates every 5 minutes (see [Collection coverage](docs/coverage.md) for which profiles and MIBs). The **BGP** page lists every session on every device, the ones that are down first, filterable by state, AS, family and text; a device's **Routing** tab shows its sessions with click-to-graph on prefixes and updates.
 
 The built-in rule `bgp_peer_down` (critical, state not established for 2 minutes) fires only where a policy binds it, like every rule: use the template "Routers: BGP peer down" under Alerts > Alert policies. Its notification names the peer and, from discovery, the peer's AS and description. A prefix-drop alert is a custom rule (v1.2); an example will be added here once custom rules are in.
+
+## Poll interval
+
+SNMP interface counters are polled every 300 s by default. An admin can set a different interval for one device (Device settings, "Poll interval", 30 to 3600 s; blank returns it to the default) or through `PUT /api/v1/devices/{slug}` with `poll_interval_s`. The poller picks it up at once, no restart. Graph rate windows, reachability fallbacks and the rollup follow the device's own interval (see `docs/DEVIATIONS.md` row 49). ICMP is still probed every 15 s.
+
+**Port totals.** A port graph (`GET /api/v1/graph`) also returns `totals`: bytes in and out over the requested range, the average-rate series integrated over each point's own step (hourly and raw data stitched without double counting). `covered_seconds` against `range_seconds` says how much of the range had data; the UI marks a total with `*` when coverage is below 99.5%. Unreliable (32-bit fast link) ports get none. The port grid shows `total` beside avg and p95; the full graph has a Total column. Per port only.
+
+**Port graph parity with LibreNMS.** The port summary and the full graph show, per direction, Now / Avg / Max / 95th / Total. Max is the highest *plotted* point (with `max=true` the finer "peak within a step" is a separate Peak column). The aggregate 95th (`p95.aggregate_bps`) is the 95th percentile of in+out per 5-minute slot, counted over slots where both directions have a sample; it is live from the hot store (also next to a materialised monthly figure) and is absent when only one direction was asked for or the range has no raw data. When the port speed is known, the full legend prints each rate's share of that speed, and a toggle plots the y axis as percent of speed (hidden for unknown or zero speed). "Compare previous period" (device port row and graph view) fetches the same-length range just before the current one with the ordinary `from`/`to` of `GET /api/v1/graph`, shifts it forward onto the current axis and draws it faded and dashed, with its own avg / max / 95th / total rows in the legend; it is off while a p95 month is selected.
+
+The graph-type selector offers **Bits**, **Errors** and **Discards**. Errors and discards come from `GET /api/v1/graph/counters?iface=&kind=errors|discards`, the per-second rate of `ifInErrors`/`ifOutErrors` or `ifInDiscards`/`ifOutDiscards` with the same step and rate-window rules as bits. These counters have **no long-store rollup**, so the endpoint answers from the hot store only; a range that starts before hot retention is cut to its start (`clamped: true`, and the UI says so). **Packets** (unicast pps) and **Broadcast/Multicast** appear disabled: the generic profile does not poll `ifHC*UcastPkts`, `ifHC*MulticastPkts` or `ifHC*BroadcastPkts`. Adding them means polling three more columns on every port of every device and regenerating every fixture, so it is a separate decision (DEVIATIONS 56).
+
+The default itself is `poller.interval` in `config.yaml` (Settings, General, "Poll interval"; 30s to 1h, default 5m). A change applies without a restart to the poller, reachability, the rollup and the API. The dev flag `nmon serve --poll-interval`, when given, wins over the setting.
+
+## Testing and debugging a device's polling
+
+Three admin tools answer "why is this device not polling right", from the device page (Actions menu) or the API. None of them shows or logs a credential: the trace names the SNMP version, never the community or the v3 secrets.
+
+- **Poll now** (`POST /api/v1/devices/{slug}/poll-now`, 202) runs the device's primary group right away, outside its schedule. The result lands in the normal places (graphs, reachability). It answers 409 when the device is retired, disabled, has no SNMP credential, or is not loaded by the poller yet.
+- **Test poll** (`POST /api/v1/devices/{slug}/poll-test`, optional `?interval_seconds=300` for one interval group) runs one synchronous cycle of the device's real poll plan: the stored credential, the resolved profile, the address and port, and the same fetch code the poller uses (the varbind count starts at the default and halves on tooBig or a lost response, exactly as in polling, but is forgotten afterwards). It writes **no metrics** and changes no poller state, so it cannot cause a reboot event, a backoff or a down alert. The answer lists each request (the session, number of varbinds, first OID, start and duration in ms, outcome `ok`, `timeout`, `too_big`, `snmp_error`, `dial_error` or `error`, and whether it was a retry in smaller pieces) and every requested OID with the series it feeds, its label (interface ULID or component), the SNMP type, the raw value and what the poller would store, or why not: `no_such_object`, `no_such_instance`, `end_of_mib`, `skipped` (text, type mismatch, not available), `not_fetched` (its request failed) or `oid_mismatch`. The `duration_ms` of a request includes the SNMP library's own retries; they are not listed separately. Limits: one test per device at a time (409), four at once overall (429), 60 s at most, 5,000 OID rows at most (`truncated: true`; failures and misses are kept first). Every test is audited as `device.poll_test`.
+- **SNMP debug** (`PUT /api/v1/devices/{slug}/snmp-debug` with `{"minutes": N}`; 0 turns it off, 60 at most; `GET` reads it) records every scheduled cycle of one device while it is on: the same request and OID detail as a test poll, for the last 20 cycles (500 OID rows each, failures first), newest first. It also logs one INFO line per cycle (`snmp debug cycle`, with the counts, duration and error). It switches off by itself when the minutes are up. **Recordings are kept in memory only** and are cleared on restart; starting a new recording drops the previous one, and switching off keeps what was captured so you can still read it. Turning it on or off is audited as `device.snmp_debug`. Use Poll now while it is on to get a cycle at once.
+
+From a shell: `nmon device poll <slug> --api URL --token TOKEN` queues a poll now; add `--test` for the traced test poll (`--interval SECONDS` for one group, `--json` for the full trace).
+
+Reading a trace: a request that times out with every OID `not_fetched` is a reachability, community or ACL problem (check the target address, the SNMP version and the device's allowed managers). A `too_big` request followed by `retry` requests means the device cannot answer 40 varbinds at once; the poller adapts, so this is only a note. Many `no_such_instance` rows for interface OIDs mean the stored interfaces no longer match the device: run Rediscover.

docs/profiles.md

@@ -0,0 +1,67 @@
+# Adding a device profile
+
+A **profile** tells nmon what to collect from one kind of device. nmon ships profiles for common gear (see [Collection coverage](/help/coverage)). When your device is only partly covered, or not at all, write one. Profiles are YAML files; nmon never reads a MIB while polling, so every metric is a numeric OID.
+
+## Where profiles live
+
+Shipped profiles are built into the binary. Your own go in `<data-dir>/profiles/*.yaml`; they are loaded over the shipped ones, and a profile with the same `name` replaces the shipped one. Restart `nmon serve` after changing a file, and run `nmon profile test` first to catch mistakes: a profile that fails validation is refused with an error naming the file.
+
+## Basics
+
+    name: acme-switch
+    extends: generic
+    priority: 20
+    match:
+      sysObjectID: 1.3.6.1.4.1.99999
+      sysDescr: '(?i)^AcmeOS'
+    interfaces:
+      exclude_ifname: ['^Vlan', '^Tunnel']
+    metrics:
+      - name: acmeCpuLoad
+        category: health
+        label: "CPU load (5 min)"
+        oid: 1.3.6.1.4.1.99999.1.1.5
+        type: gauge
+        unit: percent
+        interval: 5m
+        index_label: { column: acmeCpuIndex, as: component }
+
+- **extends: generic** inherits the standard metrics (interfaces, sysUpTime, host resources, entity sensors, LLDP). Always extend `generic`; your file only adds what is specific to the device.
+- **match** picks the profile for a device at discovery. `sysObjectID` is an OID prefix (the vendor's enterprise number, or a product branch of it). `sysDescr` is a regular expression on the system description. Give either or both; both must match when both are present.
+- **priority**: when several profiles match, the highest priority wins. `generic` is 0. Give a model-specific profile a higher number than the vendor-wide one, and watch overlaps: many devices run Linux and share its `sysDescr`, and several products share one enterprise number.
+- **metrics**: use numeric OIDs, taken from the vendor MIB, and keep a comment naming where each came from.
+
+## Metric fields
+
+| Field | Meaning |
+|---|---|
+| `name` | Series name stored in the metric store. |
+| `oid` | Numeric OID of the column or scalar. |
+| `type` | `counter64`, `counter32`, `gauge`, `status` or `text`. |
+| `unit` | `percent`, `celsius`, `octets`, `bytes`, `enum`, `count` and so on. |
+| `interval` | How often to poll, for example `60s` or `5m`. |
+| `category` | Where it shows on the device page, for example `health`, `routing`. Every metric that is not an interface metric needs one. |
+| `label` | The human-readable name shown in the UI. |
+| `index_label` | Which table column identifies the row, and the label it becomes (`component`, or `iface` for interfaces). |
+| `scale` | Multiplier for the raw value, for example `1024` to turn kilobytes into bytes. |
+| `enum` | For `status` metrics, the value-to-name map. |
+
+Every port is polled. Only `interfaces.exclude_ifname` (a list of regular expressions on the interface name) drops one. 32-bit counters on fast links are flagged unreliable and left out of totals and rollups.
+
+## Testing a profile
+
+    nmon profile test 2001:db8::10 --community public --data-dir /var/lib/nmon
+
+`profile test` runs discovery and one poll against a live device with the matching profile and prints every metric with its resolved unit, so you can check the match, the OIDs and the units before the device is added. Use `--version 3 --v3-user USER ...` for SNMPv3 (`nmon profile test --help` lists every option). The `--data-dir` flag makes it load your profiles from `<data-dir>/profiles`.
+
+## Recording a device
+
+    nmon profile record 2001:db8::10 --community public --out acme.snmprec
+
+`profile record` does a read-only SNMP walk and writes a `.snmprec` file with communities, `sysContact` and `sysLocation` redacted. Keep it: it is the best bug report for a device nmon does not understand, and it lets a profile be tested without the hardware.
+
+## Drafting metrics from a MIB
+
+    nmon profile new --mib ACME-SYSTEM-MIB
+
+`profile new` compiles objects from a MIB into draft metric entries with the OID, type and enum filled in, leaving `unit`, `category`, `label` and `interval` for you. nmon ships the standard MIBs; drop a vendor MIB into the `mibs/` folder (the `--mibs-dir` setting, by default `<data-dir>/mibs`) first. Review the draft, then test it with `nmon profile test`.

internal/alert/evaluator.go

@@ -81,6 +81,10 @@ type Options struct {
 	Notifier Notifier
 	Clock    clock.Clock
 	Rules    []Rule
+	// RuleSet, when set, is asked for the rules at the start of every cycle, so rules edited at runtime apply
+	// on the next cycle without a restart. A rule that no longer compiles is skipped with a warning, which
+	// retires its open alerts silently. Rules then only seeds the first cycle's validation and may be empty.
+	RuleSet func() []Rule
 
 	// Groups answers group membership for Rule.Overrides. Optional.
 	Groups GroupLookup
@@ -115,6 +119,7 @@ type Evaluator struct {
 	rules []*compiledRule
 	log   *slog.Logger
 	reg   *selfmon.Registry
+	bad   map[string]string // rule -> last compile error logged, so a broken rule warns once per change
 
 	mu sync.Mutex
 }
@@ -136,11 +141,27 @@ func New(o Options) (*Evaluator, error) {
 	if o.Metrics == nil {
 		o.Metrics = selfmon.Default
 	}
-	ev := &Evaluator{o: o, log: o.Logger.With("component", "alert"), reg: o.Metrics}
+	ev := &Evaluator{o: o, log: o.Logger.With("component", "alert"), reg: o.Metrics, bad: map[string]string{}}
+	rules, err := ev.compileAll(o.Rules, true)
+	if err != nil {
+		return nil, err
+	}
+	ev.rules = rules
+	return ev, nil
+}
+
+// compileAll compiles the enabled rules. Strict mode (New) fails on the first duplicate or invalid rule; the
+// lenient mode used on every cycle skips what does not compile, so one bad custom rule cannot stop the others.
+func (ev *Evaluator) compileAll(rs []Rule, strict bool) ([]*compiledRule, error) {
+	var out []*compiledRule
 	seen := map[string]bool{}
-	for _, r := range o.Rules {
+	for _, r := range rs {
 		if seen[r.Name] {
-			return nil, fmt.Errorf("alert: duplicate rule %q", r.Name)
+			if strict {
+				return nil, fmt.Errorf("alert: duplicate rule %q", r.Name)
+			}
+			ev.warnBad(r.Name, "duplicate rule name")
+			continue
 		}
 		seen[r.Name] = true
 		if r.Disabled {
@@ -148,12 +169,33 @@ func New(o Options) (*Evaluator, error) {
 		}
 		c, err := compile(r)
 		if err != nil {
-			return nil, err
+			if strict {
+				return nil, err
+			}
+			ev.warnBad(r.Name, err.Error())
+			continue
 		}
-		ev.rules = append(ev.rules, c)
+		delete(ev.bad, r.Name)
+		out = append(out, c)
 		ev.reg.Counter("nmon_alert_eval_errors_total", "rule", r.Name) // visible at 0
 	}
-	return ev, nil
+	return out, nil
+}
+
+func (ev *Evaluator) warnBad(name, msg string) {
+	if ev.bad[name] != msg {
+		ev.bad[name] = msg
+		ev.log.Warn("alert rule skipped", "rule", name, "err", msg)
+	}
+}
+
+// refreshRules re-reads the rule set from Options.RuleSet.
+func (ev *Evaluator) refreshRules() {
+	if ev.o.RuleSet == nil {
+		return
+	}
+	rules, _ := ev.compileAll(ev.o.RuleSet(), false)
+	ev.rules = rules
 }
 
 // Run evaluates once now and then every Interval until ctx ends.
@@ -181,6 +223,7 @@ func (ev *Evaluator) EvalOnce(ctx context.Context) {
 	ev.mu.Lock()
 	defer ev.mu.Unlock()
 	now := ev.o.Clock.Now().UTC()
+	ev.refreshRules()
 
 	rows, err := ev.o.Store.ListActiveAlertStates(ctx)
 	if err != nil {
@@ -265,6 +308,8 @@ func entityOf(a *store.Alert) Entity {
 
 // SortedRuleNames lists the enabled rules, for diagnostics.
 func (ev *Evaluator) SortedRuleNames() []string {
+	ev.mu.Lock()
+	defer ev.mu.Unlock()
 	var n []string
 	for _, r := range ev.rules {
 		n = append(n, r.Name)

internal/alert/integration_rules_test.go

@@ -0,0 +1,98 @@
+//go:build integration && linux
+
+package alert
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/alertrule"
+	"github.com/v10networks/nmon/internal/metrics"
+	"github.com/v10networks/nmon/internal/vmsup"
+)
+
+// TestIntegrationBuilderTemplatesAgainstRealStore compiles every combination of transform, function and
+// aggregate the builder allows for a catalogue of representative metrics and runs each query through the real
+// hot store: all must be accepted and, with data present, return readings.
+func TestIntegrationBuilderTemplatesAgainstRealStore(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+	sup := startHotStore(ctx, t)
+
+	end := time.Now().Add(-2 * time.Minute).Truncate(time.Minute)
+	var ss []metrics.Sample
+	for i := 0; i <= 30; i++ {
+		ts := end.Add(-time.Duration(30-i) * time.Minute)
+		add := func(name string, v float64, kv ...string) { ss = append(ss, metrics.New(name, v, ts, kv...)) }
+		add("hrProcessorLoad", 96, "device", "cpu1", "component", "1")
+		add("hrProcessorLoad", 40, "device", "cpu1", "component", "2")
+		add("hrStorageUsed", 900, "device", "disk1", "component", "/data")
+		add("hrStorageSize", 1000, "device", "disk1", "component", "/data")
+		add("ifInErrors", float64(i)*60, "device", "sw1", "iface", "if1")
+		add("ifHCInOctets", float64(i)*60*1000, "device", "sw1", "iface", "if1")
+		add("ifOperStatus", 2, "device", "sw1", "iface", "if1")
+		add("memTotalReal", 1000, "device", "lin1")
+		add("room_temp", 31, "device", "room1", "push", "1")
+	}
+	q := vmsup.NewQueue(vmsup.StoreHot, sup, vmsup.QueueOptions{MaxAge: 72 * time.Hour})
+	if err := q.Write(ctx, ss); err != nil {
+		t.Fatal(err)
+	}
+	if err := q.Flush(ctx); err != nil {
+		t.Fatal(err)
+	}
+	cat := alertrule.NewCatalogue([]alertrule.MetricInfo{
+		{Name: "hrProcessorLoad", Type: "gauge", Level: alertrule.LevelComponent, IntervalSeconds: 60},
+		{Name: "hrStorageUsed", Type: "gauge", Level: alertrule.LevelComponent, Of: "hrStorageSize", IntervalSeconds: 60},
+		{Name: "hrStorageSize", Type: "gauge", Level: alertrule.LevelComponent, IntervalSeconds: 60},
+		{Name: "ifInErrors", Type: "counter32", Level: alertrule.LevelInterface, IntervalSeconds: 60},
+		{Name: "ifHCInOctets", Type: "counter64", Level: alertrule.LevelInterface, IntervalSeconds: 60},
+		{Name: "ifOperStatus", Type: "status", Level: alertrule.LevelInterface, IntervalSeconds: 60},
+		{Name: "memTotalReal", Type: "gauge", Level: alertrule.LevelDevice, IntervalSeconds: 60},
+		{Name: "room_temp", Type: "gauge", Level: alertrule.LevelDevice, Source: alertrule.SourcePush},
+	})
+	var specs []alertrule.MetricSpec
+	for _, m := range cat.List() {
+		for _, tr := range m.Transforms {
+			funcs := []string{"last", "avg", "max", "min"}
+			if tr == alertrule.TransformRate {
+				funcs = []string{"avg"}
+			}
+			aggs := []string{"each"}
+			if m.Level == alertrule.LevelComponent {
+				aggs = append(aggs, "max", "min", "avg", "sum")
+			}
+			for _, f := range funcs {
+				for _, a := range aggs {
+					specs = append(specs, alertrule.MetricSpec{Metric: m.Name, Transform: tr, Func: f, Aggregate: a, Window: "10m", Op: ">=", Threshold: -1e18})
+				}
+			}
+		}
+	}
+	// Retry until the asynchronous ingest makes the data searchable.
+	deadline := time.Now().Add(40 * time.Second)
+	for _, sp := range specs {
+		for {
+			c, err := alertrule.CompileMetric(&sp, cat, time.Minute)
+			if err != nil {
+				t.Fatalf("%+v: %v", sp, err)
+			}
+			rule := Rule{Name: "t", Severity: SevInfo, Expr: c.Expr, Op: c.Op, Threshold: c.Threshold, Labels: c.Labels, Summary: "s", Description: "d"}
+			rds, err := Probe(ctx, vmsup.NewQuerier(sup), StoreHot, rule, time.Now())
+			if err == nil && len(rds) > 0 {
+				for _, rd := range rds {
+					if !rd.Breach {
+						t.Errorf("%s: %+v does not breach threshold -1e18", fmt.Sprint(sp), rd)
+					}
+				}
+				break
+			}
+			if time.Now().After(deadline) {
+				t.Fatalf("%s %+v: no readings (err %v)", c.Expr, sp, err)
+			}
+			time.Sleep(time.Second)
+		}
+	}
+}

internal/alert/integration_test.go

@@ -21,25 +21,9 @@ import (
 // the real hot store with synthetic samples: each query must be accepted, and
 // the ones with data must fire.
 func TestIntegrationDefaultRulesAgainstRealStore(t *testing.T) {
-	bin, err := filepath.Abs(filepath.Join("..", "vmsup", "bin", "victoria-metrics-prod-linux-"+runtime.GOARCH))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err := os.Stat(bin); err != nil {
-		t.Skipf("store binary not found (%v); run scripts/fetch-vm.sh", err)
-	}
-	t.Setenv("NMON_VM_BINARY", bin)
-	t.Setenv("NMON_VM_BINARY_UNVERIFIED", "")
-	sup, err := vmsup.New(vmsup.Options{DataDir: t.TempDir()})
-	if err != nil {
-		t.Fatal(err)
-	}
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancel()
-	if err := sup.Start(ctx); err != nil {
-		t.Fatal(err)
-	}
-	t.Cleanup(func() { _ = sup.Stop(context.Background()) })
+	sup := startHotStore(ctx, t)
 
 	// Samples every minute for 7 h, ending 2 min ago (the store hides the
 	// newest seconds from instant queries).
@@ -154,3 +138,26 @@ func TestIntegrationDefaultRulesAgainstRealStore(t *testing.T) {
 		}
 	}
 }
+
+// startHotStore runs a real embedded VictoriaMetrics pair for the test, or skips when the binary is absent.
+func startHotStore(ctx context.Context, t *testing.T) *vmsup.Supervisor {
+	t.Helper()
+	bin, err := filepath.Abs(filepath.Join("..", "vmsup", "bin", "victoria-metrics-prod-linux-"+runtime.GOARCH))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(bin); err != nil {
+		t.Skipf("store binary not found (%v); run scripts/fetch-vm.sh", err)
+	}
+	t.Setenv("NMON_VM_BINARY", bin)
+	t.Setenv("NMON_VM_BINARY_UNVERIFIED", "")
+	sup, err := vmsup.New(vmsup.Options{DataDir: t.TempDir()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := sup.Start(ctx); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = sup.Stop(context.Background()) })
+	return sup
+}

internal/alert/liverules_test.go

@@ -0,0 +1,111 @@
+package alert
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+)
+
+// staticSource is an inventory rule's source: fixed readings.
+type staticSource struct {
+	mu sync.Mutex
+	rs []Reading
+}
+
+func (s *staticSource) Read(context.Context, time.Time) ([]Reading, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]Reading(nil), s.rs...), nil
+}
+
+func (s *staticSource) set(rs ...Reading) { s.mu.Lock(); s.rs = rs; s.mu.Unlock() }
+
+func TestComparisonOperators(t *testing.T) {
+	cases := []struct {
+		op   string
+		v, t float64
+		want bool
+	}{
+		{OpGreater, 2, 1, true}, {OpGreater, 1, 1, false},
+		{OpGreaterEqual, 1, 1, true}, {OpGreaterEqual, 0, 1, false},
+		{OpLess, 0, 1, true}, {OpLess, 1, 1, false},
+		{OpLessEqual, 1, 1, true}, {OpLessEqual, 2, 1, false},
+		{OpEqual, 1, 1, true}, {OpEqual, 2, 1, false},
+		{OpNotEqual, 2, 1, true}, {OpNotEqual, 1, 1, false},
+	}
+	for _, c := range cases {
+		r := Rule{Op: c.op}
+		if got := r.breach(c.v, c.t); got != c.want {
+			t.Errorf("%v %s %v = %v, want %v", c.v, c.op, c.t, got, c.want)
+		}
+	}
+	r := testRule(0)
+	r.Op = "=~"
+	if r.Validate() == nil {
+		t.Fatal("op =~ accepted")
+	}
+	for _, op := range []string{OpGreaterEqual, OpLessEqual, OpEqual, OpNotEqual} {
+		r.Op = op
+		if err := r.Validate(); err != nil {
+			t.Errorf("op %s refused: %v", op, err)
+		}
+	}
+}
+
+// TestRuleSetChangesApplyOnTheNextCycle is the live-edit contract: no restart, the next EvalOnce sees the new
+// threshold, a new rule and a removed rule.
+func TestRuleSetChangesApplyOnTheNextCycle(t *testing.T) {
+	h := newHarness(t)
+	var mu sync.Mutex
+	rules := []Rule{testRule(0)}
+	h.opts.RuleSet = func() []Rule { mu.Lock(); defer mu.Unlock(); return rules }
+	h.restart()
+	h.q.set(testExpr, "d1|/|80")
+	h.step(30 * time.Second)
+	h.want("", "")
+	mu.Lock()
+	rules[0].Threshold = 70
+	mu.Unlock()
+	h.step(30 * time.Second)
+	h.want("firing", "")
+	mu.Lock()
+	rules = nil
+	mu.Unlock()
+	h.step(30 * time.Second)
+	if names := h.ev.SortedRuleNames(); len(names) != 0 {
+		t.Fatalf("rules after removal: %v", names)
+	}
+	h.want("firing", "") // retired silently: no resolved notification
+}
+
+func TestInventoryRuleFiresForMatchingPortOnly(t *testing.T) {
+	src := &staticSource{}
+	r := Rule{Name: "uplink_no_desc", Severity: SevInfo, Op: OpGreater, Threshold: 0, Labels: []string{"device", "iface"},
+		Summary: "s", Description: "d", Source: src}
+	h := newHarness(t, r)
+	src.set(Reading{Device: "d1", Iface: "p1", Value: 1}, Reading{Device: "d1", Iface: "p2", Value: 0})
+	h.step(30 * time.Second)
+	as := h.alerts()
+	if len(as) != 1 || as[0].Iface != "p1" || as[0].State != "firing" {
+		t.Fatalf("alerts: %+v", as)
+	}
+	src.set(Reading{Device: "d1", Iface: "p1", Value: 0}, Reading{Device: "d1", Iface: "p2", Value: 0})
+	h.step(30 * time.Second)
+	h.want("firing,resolved", "")
+}
+
+func TestProbeReportsBreachWithoutState(t *testing.T) {
+	h := newHarness(t)
+	h.q.set(testExpr, "d1|/|95", "d2|/|10")
+	rds, err := Probe(context.Background(), h.q, StoreHot, testRule(0), h.clk.Now())
+	if err != nil || len(rds) != 2 {
+		t.Fatalf("%v %v", rds, err)
+	}
+	if !rds[0].Breach || rds[1].Breach {
+		t.Fatalf("breach flags: %+v", rds)
+	}
+	if n := len(h.alerts()); n != 0 {
+		t.Fatalf("probe wrote %d alert rows", n)
+	}
+}

internal/alert/query.go

@@ -13,20 +13,20 @@ import (
 
 // query runs a rule and returns one reading per active, trusted entity.
 func (ev *Evaluator) query(ctx context.Context, r *compiledRule, now time.Time) ([]reading, error) {
-	qctx, cancel := context.WithTimeout(ctx, queryTimeout)
-	defer cancel()
-	raw, err := ev.o.Querier.Query(qctx, ev.o.StoreName, r.Expr, now)
-	if err != nil {
-		return nil, err
+	var rds []Reading
+	var err error
+	if r.Source != nil {
+		rds, err = r.Source.Read(ctx, now)
+	} else {
+		rds, err = readMetric(ctx, ev.o.Querier, ev.o.StoreName, r, now)
 	}
-	series, err := parseVector(raw)
 	if err != nil {
 		return nil, err
 	}
 	best := map[string]reading{}
 	var order []string
-	for _, s := range series {
-		ent, bind, ok := ev.entityOfSeries(r, s)
+	for _, s := range rds {
+		ent, bind, ok := ev.entityOfReading(r, s)
 		if !ok {
 			continue
 		}
@@ -36,8 +36,8 @@ func (ev *Evaluator) query(ctx context.Context, r *compiledRule, now time.Time)
 			order = append(order, k)
 		}
 		// Duplicate series for one entity: keep the one nearer to alerting.
-		if !seen || r.nearer(s.value, cur.value) {
-			best[k] = reading{ent, s.value, bind}
+		if !seen || r.nearer(s.Value, cur.value) {
+			best[k] = reading{ent, s.Value, bind}
 		}
 	}
 	out := make([]reading, 0, len(order))
@@ -47,33 +47,57 @@ func (ev *Evaluator) query(ctx context.Context, r *compiledRule, now time.Time)
 	return out, nil
 }
 
+// readMetric runs the rule's expression and maps each series to the entity its labels name. Series lacking a
+// label the rule needs are dropped.
+func readMetric(ctx context.Context, q Querier, storeName string, r *compiledRule, now time.Time) ([]Reading, error) {
+	qctx, cancel := context.WithTimeout(ctx, queryTimeout)
+	defer cancel()
+	raw, err := q.Query(qctx, storeName, r.Expr, now)
+	if err != nil {
+		return nil, err
+	}
+	series, err := parseVector(raw)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]Reading, 0, len(series))
+	for _, s := range series {
+		rd := Reading{Value: s.value}
+		complete := true
+		for _, l := range r.Labels {
+			v := s.labels[l]
+			if v == "" {
+				complete = false
+				break
+			}
+			switch l {
+			case "device":
+				rd.Device = v
+			case "iface":
+				rd.Iface = v
+			case "component":
+				rd.Component = v
+			}
+		}
+		if complete {
+			out = append(out, rd)
+		}
+	}
+	return out, nil
+}
+
 // nearer reports whether a is closer to breaching than b.
 func (r *compiledRule) nearer(a, b float64) bool {
-	if r.Op == OpLess {
+	if r.Op == OpLess || r.Op == OpLessEqual {
 		return a < b
 	}
 	return a > b
 }
 
-// entityOfSeries maps a series to its entity. It reports false for series that
-// lack a label the rule needs, are on unreliable interfaces, are inactive or that no alert policy binds.
-func (ev *Evaluator) entityOfSeries(r *compiledRule, s sample) (Entity, alertpolicy.Binding, bool) {
-	ent := Entity{Rule: r.Name}
-	for _, l := range r.Labels {
-		v := s.labels[l]
-		if v == "" {
-			ev.log.Debug("series without label", "rule", r.Name, "label", l)
-			return Entity{}, alertpolicy.Binding{}, false
-		}
-		switch l {
-		case "device":
-			ent.Device = v
-		case "iface":
-			ent.Iface = v
-		case "component":
-			ent.Component = v
-		}
-	}
+// entityOfReading maps a reading to its entity. It reports false for readings on unreliable interfaces, for
+// inactive entities and for entities no alert policy binds.
+func (ev *Evaluator) entityOfReading(r *compiledRule, s Reading) (Entity, alertpolicy.Binding, bool) {
+	ent := Entity{Rule: r.Name, Device: s.Device, Iface: s.Iface, Component: s.Component}
 	if r.SkipUnreliable && ent.Iface != "" && ev.o.Unreliable != nil && ev.o.Unreliable(ent.Iface) {
 		return Entity{}, alertpolicy.Binding{}, false
 	}

internal/alert/rules.go

@@ -25,10 +25,23 @@ const (
 
 // Comparison operators.
 const (
-	OpGreater = ">"
-	OpLess    = "<"
+	OpGreater      = ">"
+	OpLess         = "<"
+	OpGreaterEqual = ">="
+	OpLessEqual    = "<="
+	OpEqual        = "=="
+	OpNotEqual     = "!="
 )
 
+// validOp reports whether op is one of the six comparisons.
+func validOp(op string) bool {
+	switch op {
+	case OpGreater, OpLess, OpGreaterEqual, OpLessEqual, OpEqual, OpNotEqual:
+		return true
+	}
+	return false
+}
+
 // Rule is one metric alert rule. Expr is an instant MetricsQL query that
 // returns one series per entity whose value is the measured quantity; the
 // evaluator compares it with the threshold (so thresholds can differ per group
@@ -38,7 +51,7 @@ type Rule struct {
 	Name     string `yaml:"name" json:"name"`
 	Severity string `yaml:"severity" json:"severity"`
 	Expr     string `yaml:"expr" json:"expr"`
-	Op       string `yaml:"op" json:"op"` // ">" or "<"
+	Op       string `yaml:"op" json:"op"` // > >= < <= == !=
 	// Threshold is the default; see Overrides and Evaluator Options.ThresholdFor.
 	Threshold float64       `yaml:"threshold" json:"threshold"`
 	For       time.Duration `yaml:"for" json:"for"`
@@ -57,6 +70,8 @@ type Rule struct {
 	// a rule fires only where a policy binds it.
 	OptIn    bool `yaml:"opt_in" json:"opt_in"`
 	Disabled bool `yaml:"disabled" json:"disabled"`
+	// Source, when set, supplies the readings instead of Expr (inventory rules, evaluated in Go). Expr is then empty.
+	Source Source `yaml:"-" json:"-"`
 }
 
 // Override replaces a rule's threshold for devices in a group.
@@ -97,8 +112,8 @@ func (r Rule) Validate() error {
 	default:
 		add("severity %q is not critical, warning or info", r.Severity)
 	}
-	if r.Op != OpGreater && r.Op != OpLess {
-		add("op %q is not > or <", r.Op)
+	if !validOp(r.Op) {
+		add("op %q is not one of > >= < <= == !=", r.Op)
 	}
 	if r.For < 0 {
 		add("for is negative")
@@ -107,8 +122,12 @@ func (r Rule) Validate() error {
 		add("name is reserved for reachability alerts")
 	}
 	r.validateLabels(add)
-	if err := checkExprShape(r.Expr); err != nil {
-		add("expr: %v", err)
+	if r.Source == nil {
+		if err := checkExprShape(r.Expr); err != nil {
+			add("expr: %v", err)
+		}
+	} else if r.Expr != "" {
+		add("a rule with a source takes no expr")
 	}
 	r.validateOverridesAndTemplates(add)
 	return errors.Join(errs...)
@@ -213,8 +232,17 @@ func (c *compiledRule) render(t *template.Template, d TemplateData) string {
 
 // breach reports whether v is on the alerting side of threshold.
 func (r Rule) breach(v, threshold float64) bool {
-	if r.Op == OpLess {
+	switch r.Op {
+	case OpLess:
 		return v < threshold
+	case OpLessEqual:
+		return v <= threshold
+	case OpGreaterEqual:
+		return v >= threshold
+	case OpEqual:
+		return v == threshold
+	case OpNotEqual:
+		return v != threshold
 	}
 	return v > threshold
 }
@@ -394,11 +422,11 @@ func hostRules(lab func(...string) []string) []Rule {
 		{
 			Name: "interface_errors", Severity: SevWarning, Op: OpGreater, Threshold: 0.1, For: 0,
 			Labels: lab("device", "iface"), SkipUnreliable: true,
-			Expr: `100 * (rate(ifInErrors[5m]) + rate(ifOutErrors[5m])) / clamp_min(` +
-				`(rate(ifHCInUcastPkts[5m]) + rate(ifHCOutUcastPkts[5m])) or ` +
-				`((rate(ifHCInOctets[5m]) + rate(ifHCOutOctets[5m])) / 1500), 10)`,
+			Expr: `100 * (rate(ifInErrors[10m]) + rate(ifOutErrors[10m])) / clamp_min(` +
+				`(rate(ifHCInUcastPkts[10m]) + rate(ifHCOutUcastPkts[10m])) or ` +
+				`((rate(ifHCInOctets[10m]) + rate(ifHCOutOctets[10m])) / 1500), 10)`,
 			Summary: `{{.Device}} interface {{.Iface}} error ratio {{printf "%.2f" .Value}}%`,
-			Description: `Errors are {{printf "%.2f" .Value}}% of packets over 5 minutes on {{.Iface}} of {{.Device}} ` +
+			Description: `Errors are {{printf "%.2f" .Value}}% of packets over 10 minutes on {{.Iface}} of {{.Device}} ` +
 				`(threshold {{printf "%.2f" .Threshold}}%).`,
 		},
 		{

internal/alert/source.go

@@ -0,0 +1,49 @@
+package alert
+
+import (
+	"context"
+	"time"
+)
+
+// Reading is one entity's measured value, as a Source reports it.
+type Reading struct {
+	Device, Iface, Component string
+	Value                    float64
+}
+
+// Source supplies a rule's readings from somewhere other than the metric store. The app implements it for
+// inventory rules, which are evaluated in Go over device and interface rows. Like a query result it must
+// include entities that do not breach (value below the threshold), because a missing entity is "no data", not
+// "resolved".
+type Source interface {
+	Read(ctx context.Context, now time.Time) ([]Reading, error)
+}
+
+// ProbeReading is a reading and whether it breaches the rule's own threshold.
+type ProbeReading struct {
+	Reading
+	Breach bool
+}
+
+// Probe runs one rule once, without touching alert state or any policy, and reports what it sees. The API's
+// rule test uses it to show an admin what a draft rule would do.
+func Probe(ctx context.Context, q Querier, storeName string, r Rule, now time.Time) ([]ProbeReading, error) {
+	c, err := compile(r)
+	if err != nil {
+		return nil, err
+	}
+	var rds []Reading
+	if r.Source != nil {
+		rds, err = r.Source.Read(ctx, now)
+	} else {
+		rds, err = readMetric(ctx, q, storeName, c, now)
+	}
+	if err != nil {
+		return nil, err
+	}
+	out := make([]ProbeReading, 0, len(rds))
+	for _, rd := range rds {
+		out = append(out, ProbeReading{Reading: rd, Breach: c.breach(rd.Value, r.Threshold)})
+	}
+	return out, nil
+}

internal/alertrule/catalogue.go

@@ -0,0 +1,101 @@
+package alertrule
+
+import (
+	"regexp"
+	"sort"
+)
+
+// Sources of a catalogue metric.
+const (
+	SourceProfile = "profile"
+	SourcePush    = "push"
+)
+
+// Levels of a metric rule, equal to the alert policy levels.
+const (
+	LevelDevice    = "device"
+	LevelInterface = "interface"
+	LevelComponent = "component"
+)
+
+// MetricInfo describes one metric a rule may use.
+type MetricInfo struct {
+	Name            string         `json:"name"`
+	Label           string         `json:"label"`
+	Unit            string         `json:"unit"`
+	Type            string         `json:"type" doc:"counter64, counter32, gauge, status for profile metrics; gauge for push."`
+	Category        string         `json:"category,omitempty"`
+	Of              string         `json:"of,omitempty" doc:"The metric this one is a fraction of; it enables percent_of."`
+	Enum            map[int]string `json:"enum,omitempty"`
+	Level           string         `json:"level" enum:"device,interface,component"`
+	Source          string         `json:"source" enum:"profile,push"`
+	IntervalSeconds int64          `json:"interval_seconds,omitempty" doc:"Poll interval; the window must be at least twice this."`
+	Transforms      []string       `json:"transforms" doc:"The transforms valid for this metric."`
+}
+
+// Catalogue is the set of metrics a rule may name.
+type Catalogue struct {
+	byName map[string]MetricInfo
+}
+
+// identRE is what a metric name must look like to be put in a query. Names that do not (a push client can send
+// anything) are left out of the catalogue rather than escaped.
+var identRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
+
+// NewCatalogue indexes metrics, dropping names that are not plain identifiers. The first of a name wins. Each
+// metric's Transforms are filled in.
+func NewCatalogue(ms []MetricInfo) Catalogue {
+	c := Catalogue{byName: map[string]MetricInfo{}}
+	for _, m := range ms {
+		if !identRE.MatchString(m.Name) {
+			continue
+		}
+		if _, dup := c.byName[m.Name]; dup {
+			continue
+		}
+		c.byName[m.Name] = m
+	}
+	for n, m := range c.byName {
+		m.Transforms = c.transformsOf(m)
+		c.byName[n] = m
+	}
+	return c
+}
+
+func (c Catalogue) transformsOf(m MetricInfo) []string {
+	switch m.Type {
+	case "counter64", "counter32":
+		return []string{TransformRate}
+	}
+	out := []string{TransformValue}
+	if m.Of != "" {
+		if _, ok := c.byName[m.Of]; ok {
+			out = append(out, TransformPercentOf)
+		}
+	}
+	return out
+}
+
+// Lookup finds a metric by name.
+func (c Catalogue) Lookup(name string) (MetricInfo, bool) {
+	m, ok := c.byName[name]
+	return m, ok
+}
+
+// List returns the metrics ordered by category, then name.
+func (c Catalogue) List() []MetricInfo {
+	out := make([]MetricInfo, 0, len(c.byName))
+	for _, m := range c.byName {
+		out = append(out, m)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Category != out[j].Category {
+			return out[i].Category < out[j].Category
+		}
+		return out[i].Name < out[j].Name
+	})
+	return out
+}
+
+// Len is the number of metrics.
+func (c Catalogue) Len() int { return len(c.byName) }

internal/alertrule/fields.go

@@ -0,0 +1,91 @@
+package alertrule
+
+import "sort"
+
+// Types of an inventory field.
+const (
+	TypeNumber   = "number"
+	TypeBool     = "bool"
+	TypeString   = "string"
+	TypeDuration = "duration"
+)
+
+// Entities an inventory rule can range over; they are also the rule's level.
+const (
+	EntityDevice    = "device"
+	EntityInterface = "interface"
+)
+
+// Ops by field type.
+var typeOps = map[string][]string{
+	TypeNumber:   {OpGT, OpGE, OpLT, OpLE, OpEQ, OpNE},
+	TypeBool:     {"is_true", "is_false"},
+	TypeString:   {"equals", "not_equals", "matches", "empty", "not_empty"},
+	TypeDuration: {"older_than", "newer_than"},
+}
+
+// OpsFor lists the ops a field type accepts.
+func OpsFor(typ string) []string { return append([]string(nil), typeOps[typ]...) }
+
+// InventoryField is one whitelisted inventory field.
+type InventoryField struct {
+	Name   string         `json:"name"`
+	Label  string         `json:"label"`
+	Entity string         `json:"entity" enum:"device,interface"`
+	Type   string         `json:"type" enum:"number,bool,string,duration"`
+	Ops    []string       `json:"ops"`
+	Enum   map[int]string `json:"enum,omitempty" doc:"Names of a status number, for the UI; the value is still the number."`
+}
+
+type fieldDef struct {
+	name, label, entity, typ string
+	enum                     map[int]string
+}
+
+// The whitelist. Device rows that are retired, purged or disabled are never evaluated (their alerts close as
+// retired), so there is no enabled field. snmp_failing is true while the poller reports the device degraded or
+// backed off.
+var fieldDefs = []fieldDef{
+	{"last_seen_age", "Time since last seen", EntityDevice, TypeDuration, nil},
+	{"profile", "Profile", EntityDevice, TypeString, nil},
+	{"profile_pinned", "Profile pinned", EntityDevice, TypeBool, nil},
+	{"site", "Site", EntityDevice, TypeString, nil},
+	{"role", "Role", EntityDevice, TypeString, nil},
+	{"kind", "Kind", EntityDevice, TypeString, nil},
+	{"sys_descr", "sysDescr", EntityDevice, TypeString, nil},
+	{"hw_serial", "Hardware serial", EntityDevice, TypeString, nil},
+	{"mgmt_ip", "Management address", EntityDevice, TypeString, nil},
+	{"parent_count", "Number of parents", EntityDevice, TypeNumber, nil},
+	{"snmp_failing", "SNMP failing", EntityDevice, TypeBool, nil},
+	{"if_speed_mbps", "Port speed (Mb/s)", EntityInterface, TypeNumber, nil},
+	{"admin_status", "Admin status", EntityInterface, TypeNumber, map[int]string{1: "up", 2: "down", 3: "testing"}},
+	{"oper_status", "Oper status", EntityInterface, TypeNumber, map[int]string{
+		1: "up", 2: "down", 3: "testing", 4: "unknown", 5: "dormant", 6: "notPresent", 7: "lowerLayerDown"}},
+	{"if_name", "Port name", EntityInterface, TypeString, nil},
+	{"if_alias", "Port alias", EntityInterface, TypeString, nil},
+	{"description", "Port description", EntityInterface, TypeString, nil},
+	{"unreliable", "Counters unreliable", EntityInterface, TypeBool, nil},
+	{"has_hc_counters", "Has 64-bit counters", EntityInterface, TypeBool, nil},
+	{"excluded", "Excluded from polling", EntityInterface, TypeBool, nil},
+	{"counter_discontinuity", "Counter discontinuities", EntityInterface, TypeNumber, nil},
+	{"last_discovered_age", "Time since last discovered", EntityInterface, TypeDuration, nil},
+}
+
+// Fields lists the whitelist, device fields first.
+func Fields() []InventoryField {
+	out := make([]InventoryField, 0, len(fieldDefs))
+	for _, d := range fieldDefs {
+		out = append(out, InventoryField{Name: d.name, Label: d.label, Entity: d.entity, Type: d.typ, Ops: OpsFor(d.typ), Enum: d.enum})
+	}
+	sort.SliceStable(out, func(i, j int) bool { return out[i].Entity == EntityDevice && out[j].Entity != EntityDevice })
+	return out
+}
+
+func lookupField(name string) (fieldDef, bool) {
+	for _, d := range fieldDefs {
+		if d.name == name {
+			return d, true
+		}
+	}
+	return fieldDef{}, false
+}

internal/alertrule/inventory.go

@@ -0,0 +1,198 @@
+package alertrule
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+// Limits on user-supplied text in a condition.
+const (
+	maxValueLen = 500
+	maxRegexLen = 300
+)
+
+// Row is one device or interface as the inventory rule sees it. The app fills the maps from store rows (and
+// turns timestamps into ages) so this package needs no store and no clock. A missing key never matches.
+type Row struct {
+	Num  map[string]float64
+	Bool map[string]bool
+	Str  map[string]string
+	Dur  map[string]time.Duration
+}
+
+// InventoryRule is a validated, compiled inventory spec.
+type InventoryRule struct {
+	// Entity is device or interface; it is the rule's level.
+	Entity string
+	conds  []cond
+}
+
+type cond struct {
+	field fieldDef
+	op    string
+	num   float64
+	dur   time.Duration
+	text  string
+	re    *regexp.Regexp
+}
+
+var errNeedInventory = errors.New("inventory rule needs an inventory spec")
+
+// CompileInventory validates the conditions: one to five, known fields all on one entity, an op that suits the
+// field's type, and a value that parses (a regular expression must be RE2).
+func CompileInventory(s *InventorySpec) (*InventoryRule, error) {
+	if s == nil {
+		return nil, errNeedInventory
+	}
+	if n := len(s.Conditions); n < 1 || n > MaxConditions {
+		return nil, fmt.Errorf("an inventory rule needs 1 to %d conditions, got %d", MaxConditions, n)
+	}
+	r := &InventoryRule{}
+	var errs []error
+	for i, c := range s.Conditions {
+		cc, err := compileCond(c)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("condition %d: %w", i+1, err))
+			continue
+		}
+		if r.Entity == "" {
+			r.Entity = cc.field.entity
+		} else if r.Entity != cc.field.entity {
+			errs = append(errs, fmt.Errorf("condition %d: field %s is a %s field, but the rule is on %s fields",
+				i+1, c.Field, cc.field.entity, r.Entity))
+		}
+		r.conds = append(r.conds, cc)
+	}
+	if len(errs) > 0 {
+		return nil, errors.Join(errs...)
+	}
+	return r, nil
+}
+
+func compileCond(c InventoryCondition) (cond, error) {
+	f, ok := lookupField(c.Field)
+	if !ok {
+		return cond{}, fmt.Errorf("field %q is not an inventory field", c.Field)
+	}
+	if !slices.Contains(typeOps[f.typ], c.Op) {
+		return cond{}, fmt.Errorf("op %q does not apply to %s field %s (allowed: %s)",
+			c.Op, f.typ, f.name, strings.Join(typeOps[f.typ], ", "))
+	}
+	if utf8.RuneCountInString(c.Value) > maxValueLen {
+		return cond{}, fmt.Errorf("value is longer than %d characters", maxValueLen)
+	}
+	out := cond{field: f, op: c.Op}
+	var err error
+	switch f.typ {
+	case TypeNumber:
+		out.num, err = strconv.ParseFloat(strings.TrimSpace(c.Value), 64)
+		if err == nil && (math.IsNaN(out.num) || math.IsInf(out.num, 0)) {
+			err = errors.New("not finite")
+		}
+	case TypeDuration:
+		out.dur, err = time.ParseDuration(strings.TrimSpace(c.Value))
+		if err == nil && out.dur < 0 {
+			err = errors.New("negative")
+		}
+	case TypeString:
+		out.text = c.Value
+		if c.Op == "matches" {
+			out.re, err = compileRegex(c.Value)
+		}
+	}
+	if err != nil {
+		return cond{}, fmt.Errorf("value %q for %s: %w", c.Value, f.name, err)
+	}
+	return out, nil
+}
+
+func compileRegex(s string) (*regexp.Regexp, error) {
+	if len(s) > maxRegexLen {
+		return nil, fmt.Errorf("pattern is longer than %d characters", maxRegexLen)
+	}
+	return regexp.Compile(s)
+}
+
+// EvalInventory compiles s and tests row; use CompileInventory once when testing many rows.
+func EvalInventory(s *InventorySpec, row Row) (bool, error) {
+	r, err := CompileInventory(s)
+	if err != nil {
+		return false, err
+	}
+	return r.Match(row), nil
+}
+
+// Match reports whether every condition holds for row.
+func (r *InventoryRule) Match(row Row) bool {
+	for _, c := range r.conds {
+		if !c.holds(row) {
+			return false
+		}
+	}
+	return true
+}
+
+func (c cond) holds(row Row) bool {
+	switch c.field.typ {
+	case TypeNumber:
+		v, ok := row.Num[c.field.name]
+		return ok && compareNumber(c.op, v, c.num)
+	case TypeBool:
+		v, ok := row.Bool[c.field.name]
+		return ok && v == (c.op == "is_true")
+	case TypeString:
+		return matchString(c, row.Str[c.field.name])
+	case TypeDuration:
+		v, ok := row.Dur[c.field.name]
+		if !ok {
+			return false
+		}
+		if c.op == "older_than" {
+			return v > c.dur
+		}
+		return v < c.dur
+	}
+	return false
+}
+
+func compareNumber(op string, a, b float64) bool {
+	switch op {
+	case OpGT:
+		return a > b
+	case OpGE:
+		return a >= b
+	case OpLT:
+		return a < b
+	case OpLE:
+		return a <= b
+	case OpEQ:
+		return a == b
+	case OpNE:
+		return a != b
+	}
+	return false
+}
+
+// matchString compares case-sensitively, except equals, which ignores case like the policy matchers do.
+func matchString(c cond, v string) bool {
+	switch c.op {
+	case "equals":
+		return strings.EqualFold(v, c.text)
+	case "not_equals":
+		return !strings.EqualFold(v, c.text)
+	case "matches":
+		return c.re.MatchString(v)
+	case "empty":
+		return strings.TrimSpace(v) == ""
+	case "not_empty":
+		return strings.TrimSpace(v) != ""
+	}
+	return false
+}

internal/alertrule/inventory_test.go

@@ -0,0 +1,98 @@
+package alertrule
+
+import (
+	"testing"
+	"time"
+)
+
+func spec(cs ...InventoryCondition) *InventorySpec { return &InventorySpec{Conditions: cs} }
+
+func TestInventoryVerdicts(t *testing.T) {
+	uplinkNoDesc := spec(InventoryCondition{Field: "if_alias", Op: "empty"}, InventoryCondition{Field: "if_speed_mbps", Op: OpGE, Value: "10000"})
+	port := func(alias string, speed float64) Row {
+		return Row{Str: map[string]string{"if_alias": alias}, Num: map[string]float64{"if_speed_mbps": speed}}
+	}
+	cases := []struct {
+		name string
+		spec *InventorySpec
+		row  Row
+		want bool
+	}{
+		{"uplink without description matches", uplinkNoDesc, port("", 10000), true},
+		{"described uplink does not", uplinkNoDesc, port("to core", 10000), false},
+		{"slow port does not", uplinkNoDesc, port("", 1000), false},
+		{"not seen for an hour", spec(InventoryCondition{Field: "last_seen_age", Op: "older_than", Value: "1h"}),
+			Row{Dur: map[string]time.Duration{"last_seen_age": 2 * time.Hour}}, true},
+		{"seen a minute ago", spec(InventoryCondition{Field: "last_seen_age", Op: "older_than", Value: "1h"}),
+			Row{Dur: map[string]time.Duration{"last_seen_age": time.Minute}}, false},
+		{"newer than", spec(InventoryCondition{Field: "last_discovered_age", Op: "newer_than", Value: "1h"}),
+			Row{Dur: map[string]time.Duration{"last_discovered_age": time.Minute}}, true},
+		{"bool true", spec(InventoryCondition{Field: "unreliable", Op: "is_true"}), Row{Bool: map[string]bool{"unreliable": true}}, true},
+		{"bool false", spec(InventoryCondition{Field: "unreliable", Op: "is_false"}), Row{Bool: map[string]bool{"unreliable": true}}, false},
+		{"regex", spec(InventoryCondition{Field: "if_name", Op: "matches", Value: `^ether[0-9]+$`}),
+			Row{Str: map[string]string{"if_name": "ether12"}}, true},
+		{"equals ignores case", spec(InventoryCondition{Field: "site", Op: "equals", Value: "HQ"}), Row{Str: map[string]string{"site": "hq"}}, true},
+		{"not equals", spec(InventoryCondition{Field: "site", Op: "not_equals", Value: "hq"}), Row{Str: map[string]string{"site": "hq"}}, false},
+		{"status number", spec(InventoryCondition{Field: "oper_status", Op: OpEQ, Value: "2"}), Row{Num: map[string]float64{"oper_status": 2}}, true},
+		{"missing number never matches", spec(InventoryCondition{Field: "parent_count", Op: OpNE, Value: "5"}), Row{}, false},
+		{"parent count zero", spec(InventoryCondition{Field: "parent_count", Op: OpEQ, Value: "0"}), Row{Num: map[string]float64{"parent_count": 0}}, true},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got, err := EvalInventory(c.spec, c.row)
+			if err != nil || got != c.want {
+				t.Fatalf("got %v err %v, want %v", got, err, c.want)
+			}
+		})
+	}
+}
+
+func TestInventoryRefusals(t *testing.T) {
+	cases := map[string]*InventorySpec{
+		"nil":               nil,
+		"no conditions":     spec(),
+		"six conditions":    spec(make([]InventoryCondition, 6)...),
+		"unknown field":     spec(InventoryCondition{Field: "password", Op: "empty"}),
+		"op of wrong type":  spec(InventoryCondition{Field: "if_name", Op: OpGT, Value: "1"}),
+		"bad number":        spec(InventoryCondition{Field: "if_speed_mbps", Op: OpGT, Value: "fast"}),
+		"nan":               spec(InventoryCondition{Field: "if_speed_mbps", Op: OpGT, Value: "NaN"}),
+		"bad duration":      spec(InventoryCondition{Field: "last_seen_age", Op: "older_than", Value: "soon"}),
+		"bad regex":         spec(InventoryCondition{Field: "if_name", Op: "matches", Value: "("}),
+		"lookahead not RE2": spec(InventoryCondition{Field: "if_name", Op: "matches", Value: "(?=a)"}),
+		"mixed entities":    spec(InventoryCondition{Field: "site", Op: "empty"}, InventoryCondition{Field: "if_alias", Op: "empty"}),
+		"huge value":        spec(InventoryCondition{Field: "site", Op: "equals", Value: string(make([]byte, 600))}),
+		"huge regex":        spec(InventoryCondition{Field: "if_name", Op: "matches", Value: string(make([]rune, 400))}),
+		"negative duration": spec(InventoryCondition{Field: "last_seen_age", Op: "older_than", Value: "-1h"}),
+		"op injection":      spec(InventoryCondition{Field: "site", Op: `equals" OR 1=1`, Value: "x"}),
+		"field injection":   spec(InventoryCondition{Field: `site; DROP TABLE devices`, Op: "empty"}),
+	}
+	for name, s := range cases {
+		if name == "huge regex" {
+			s.Conditions[0].Value = string(repeat('a', 400))
+		}
+		t.Run(name, func(t *testing.T) {
+			if _, err := CompileInventory(s); err == nil {
+				t.Fatal("accepted")
+			}
+		})
+	}
+}
+
+func repeat(b byte, n int) []byte {
+	out := make([]byte, n)
+	for i := range out {
+		out[i] = b
+	}
+	return out
+}
+
+func TestInventoryLevel(t *testing.T) {
+	r, err := CompileInventory(spec(InventoryCondition{Field: "if_alias", Op: "empty"}))
+	if err != nil || r.Entity != EntityInterface {
+		t.Fatalf("%v %v", r, err)
+	}
+	r, err = CompileInventory(spec(InventoryCondition{Field: "site", Op: "empty"}))
+	if err != nil || r.Entity != EntityDevice {
+		t.Fatalf("%v %v", r, err)
+	}
+}

internal/alertrule/metric.go

@@ -0,0 +1,180 @@
+package alertrule
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"slices"
+	"strings"
+	"time"
+)
+
+// Compiled is a validated spec ready for the evaluator.
+type Compiled struct {
+	// Expr is the MetricsQL of a metric rule; empty for an inventory rule.
+	Expr string
+	// Level is device, interface or component; Labels are the series labels that identify the entity.
+	Level  string
+	Labels []string
+	Op     string
+	// Threshold is the default threshold (0 for an inventory rule, whose reading is 1 when the conditions hold).
+	Threshold float64
+	// SkipUnreliable asks the evaluator to ignore ports whose counters cannot be trusted.
+	SkipUnreliable bool
+	// Label is a short human name of what is measured, used in default summaries.
+	Label string
+}
+
+var errNeedMetric = errors.New("metric rule needs a metric spec")
+
+// CompileMetric validates m against the catalogue and builds its expression. fallbackInterval is the poll
+// interval assumed for a metric that carries none (push metrics); the window must be at least twice the
+// metric's interval. Every piece of the query is either a fixed template, a catalogue name that matches a plain
+// identifier, or a duration printed from a parsed number.
+func CompileMetric(m *MetricSpec, cat Catalogue, fallbackInterval time.Duration) (Compiled, error) {
+	if m == nil {
+		return Compiled{}, errNeedMetric
+	}
+	info, ok := cat.Lookup(m.Metric)
+	if !ok {
+		return Compiled{}, fmt.Errorf("metric %q is not in the catalogue", m.Metric)
+	}
+	if !slices.Contains(Ops, m.Op) {
+		return Compiled{}, fmt.Errorf("op %q is not one of %s", m.Op, strings.Join(Ops, " "))
+	}
+	if math.IsNaN(m.Threshold) || math.IsInf(m.Threshold, 0) {
+		return Compiled{}, errors.New("threshold is not a finite number")
+	}
+	win, err := windowOf(m.Window, info, fallbackInterval)
+	if err != nil {
+		return Compiled{}, err
+	}
+	tr, fn, err := transformOf(m, info)
+	if err != nil {
+		return Compiled{}, err
+	}
+	expr, err := buildExpr(tr, fn, win, info, cat)
+	if err != nil {
+		return Compiled{}, err
+	}
+	c := Compiled{Op: m.Op, Threshold: m.Threshold, Label: info.Label, Level: info.Level}
+	c.Expr, c.Level, err = aggregateOf(m.Aggregate, expr, info)
+	if err != nil {
+		return Compiled{}, err
+	}
+	c.Labels = labelsOf(c.Level)
+	c.SkipUnreliable = tr == TransformRate && info.Type == "counter32" && c.Level == LevelInterface
+	return c, nil
+}
+
+func windowOf(s string, info MetricInfo, fallback time.Duration) (time.Duration, error) {
+	every := time.Duration(info.IntervalSeconds) * time.Second
+	if every <= 0 {
+		every = fallback
+	}
+	w := max(DefaultWindow, 2*every) // the default grows with a slow poll interval, so it is always valid
+	if s != "" {
+		d, err := time.ParseDuration(s)
+		if err != nil {
+			return 0, fmt.Errorf("window: %w", err)
+		}
+		w = d
+	}
+	switch {
+	case w > MaxWindow:
+		return 0, fmt.Errorf("window %s is above the maximum %s", w, MaxWindow)
+	case w < 2*every:
+		return 0, fmt.Errorf("window %s is below twice the poll interval (%s)", w, 2*every)
+	case w < time.Minute:
+		return 0, errors.New("window is below 1m")
+	}
+	return w, nil
+}
+
+// transformOf checks the transform and function against the metric and returns their canonical names.
+func transformOf(m *MetricSpec, info MetricInfo) (transform, fn string, err error) {
+	transform, fn = m.Transform, m.Func
+	if transform == "" {
+		transform = TransformValue
+		if info.Type == "counter64" || info.Type == "counter32" {
+			transform = TransformRate
+		}
+	}
+	if !slices.Contains(info.Transforms, transform) {
+		return "", "", fmt.Errorf("transform %q does not apply to %s (allowed: %s)", transform, info.Name, strings.Join(info.Transforms, ", "))
+	}
+	if transform == TransformRate {
+		if fn != "" && fn != FuncAvg {
+			return "", "", errors.New("a rate averages over the window; func must be avg or empty")
+		}
+		return transform, FuncAvg, nil
+	}
+	if fn == "" {
+		fn = FuncLast
+	}
+	if !slices.Contains([]string{FuncLast, FuncAvg, FuncMax, FuncMin}, fn) {
+		return "", "", fmt.Errorf("func %q is not last, avg, max or min", fn)
+	}
+	return transform, fn, nil
+}
+
+// durationText prints a window the way MetricsQL reads it.
+func durationText(d time.Duration) string {
+	if d%time.Minute == 0 {
+		return fmt.Sprintf("%dm", int64(d/time.Minute))
+	}
+	return fmt.Sprintf("%ds", int64(d/time.Second))
+}
+
+func selector(m MetricInfo) string {
+	if m.Source == SourcePush {
+		return m.Name + `{push="1"}`
+	}
+	return m.Name
+}
+
+func windowed(fn string, m MetricInfo, win time.Duration) string {
+	if fn == "rate" {
+		return "rate(" + selector(m) + "[" + durationText(win) + "])"
+	}
+	return fn + "_over_time(" + selector(m) + "[" + durationText(win) + "])"
+}
+
+// buildExpr is the fixed template of a metric rule, before aggregation.
+func buildExpr(transform, fn string, win time.Duration, info MetricInfo, cat Catalogue) (string, error) {
+	switch transform {
+	case TransformRate:
+		return windowed("rate", info, win), nil
+	case TransformPercentOf:
+		of, ok := cat.Lookup(info.Of)
+		if !ok {
+			return "", fmt.Errorf("%s is a fraction of %q, which is not in the catalogue", info.Name, info.Of)
+		}
+		return "100 * " + windowed(fn, info, win) + " / (" + windowed(fn, of, win) + " > 0)", nil
+	}
+	return windowed(fn, info, win), nil
+}
+
+// aggregateOf wraps expr for a per-device aggregate and returns the level the rule then has.
+func aggregateOf(agg, expr string, info MetricInfo) (string, string, error) {
+	if agg == "" || agg == AggEach {
+		return expr, info.Level, nil
+	}
+	if !slices.Contains([]string{AggMax, AggMin, AggAvg, AggSum}, agg) {
+		return "", "", fmt.Errorf("aggregate %q is not each, max, min, avg or sum", agg)
+	}
+	if info.Level != LevelComponent {
+		return "", "", fmt.Errorf("aggregate applies to component metrics; %s is a %s metric", info.Name, info.Level)
+	}
+	return agg + " by (device) (" + expr + ")", LevelDevice, nil
+}
+
+func labelsOf(level string) []string {
+	switch level {
+	case LevelInterface:
+		return []string{"device", "iface"}
+	case LevelComponent:
+		return []string{"device", "component"}
+	}
+	return []string{"device"}
+}

internal/alertrule/metric_test.go

@@ -0,0 +1,148 @@
+package alertrule
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func testCatalogue() Catalogue {
+	return NewCatalogue([]MetricInfo{
+		{Name: "hrProcessorLoad", Label: "CPU load", Unit: "percent", Type: "gauge", Level: LevelComponent, Source: SourceProfile, IntervalSeconds: 300},
+		{Name: "hrStorageUsed", Label: "Storage used", Unit: "count", Type: "gauge", Level: LevelComponent, Source: SourceProfile,
+			Of: "hrStorageSize", IntervalSeconds: 300},
+		{Name: "hrStorageSize", Label: "Storage size", Unit: "count", Type: "gauge", Level: LevelComponent, Source: SourceProfile, IntervalSeconds: 300},
+		{Name: "ifInErrors", Label: "In errors", Unit: "errors", Type: "counter32", Level: LevelInterface, Source: SourceProfile, IntervalSeconds: 60},
+		{Name: "ifHCInOctets", Label: "In octets", Unit: "octets", Type: "counter64", Level: LevelInterface, Source: SourceProfile, IntervalSeconds: 60},
+		{Name: "ifOperStatus", Label: "Oper status", Unit: "enum", Type: "status", Level: LevelInterface, Source: SourceProfile, IntervalSeconds: 60},
+		{Name: "memTotalReal", Label: "Memory total", Unit: "bytes", Type: "gauge", Level: LevelDevice, Source: SourceProfile, IntervalSeconds: 300},
+		{Name: "room_temp", Label: "room_temp", Unit: "", Type: "gauge", Level: LevelDevice, Source: SourcePush},
+		{Name: `evil"} or vector(1) or x{a="`, Type: "gauge", Level: LevelDevice, Source: SourcePush},
+	})
+}
+
+func TestCompileMetricTemplates(t *testing.T) {
+	cat := testCatalogue()
+	cases := []struct {
+		name   string
+		spec   MetricSpec
+		expr   string
+		level  string
+		labels string
+		skip   bool
+	}{
+		{"gauge last", MetricSpec{Metric: "memTotalReal", Op: OpLT, Threshold: 1},
+			`last_over_time(memTotalReal[10m])`, LevelDevice, "device", false},
+		{"gauge max 15m", MetricSpec{Metric: "memTotalReal", Func: FuncMax, Window: "15m", Op: OpGE, Threshold: 1},
+			`max_over_time(memTotalReal[15m])`, LevelDevice, "device", false},
+		{"component each", MetricSpec{Metric: "hrProcessorLoad", Func: FuncAvg, Op: OpGT, Threshold: 80},
+			`avg_over_time(hrProcessorLoad[10m])`, LevelComponent, "device,component", false},
+		{"component aggregate", MetricSpec{Metric: "hrProcessorLoad", Aggregate: AggMax, Window: "10m", Op: OpGT, Threshold: 80},
+			`max by (device) (last_over_time(hrProcessorLoad[10m]))`, LevelDevice, "device", false},
+		{"percent_of", MetricSpec{Metric: "hrStorageUsed", Transform: TransformPercentOf, Op: OpGT, Threshold: 90},
+			`100 * last_over_time(hrStorageUsed[10m]) / (last_over_time(hrStorageSize[10m]) > 0)`, LevelComponent, "device,component", false},
+		{"rate counter32 iface", MetricSpec{Metric: "ifInErrors", Op: OpGT, Threshold: 1},
+			`rate(ifInErrors[5m])`, LevelInterface, "device,iface", true},
+		{"rate counter64 iface 90s", MetricSpec{Metric: "ifHCInOctets", Transform: TransformRate, Window: "2m", Op: OpGT, Threshold: 1e6},
+			`rate(ifHCInOctets[2m])`, LevelInterface, "device,iface", false},
+		{"status", MetricSpec{Metric: "ifOperStatus", Op: OpEQ, Threshold: 2},
+			`last_over_time(ifOperStatus[5m])`, LevelInterface, "device,iface", false},
+		{"push", MetricSpec{Metric: "room_temp", Op: OpNE, Threshold: 0},
+			`last_over_time(room_temp{push="1"}[5m])`, LevelDevice, "device", false},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got, err := CompileMetric(&c.spec, cat, time.Minute)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got.Expr != c.expr || got.Level != c.level || strings.Join(got.Labels, ",") != c.labels || got.SkipUnreliable != c.skip {
+				t.Fatalf("got %q level %s labels %v skip %v\nwant %q level %s labels %s skip %v",
+					got.Expr, got.Level, got.Labels, got.SkipUnreliable, c.expr, c.level, c.labels, c.skip)
+			}
+		})
+	}
+}
+
+func TestCompileMetricRefusals(t *testing.T) {
+	cat := testCatalogue()
+	cases := map[string]MetricSpec{
+		"unknown metric":               {Metric: "nope", Op: OpGT},
+		"bad op":                       {Metric: "memTotalReal", Op: "=~"},
+		"counter as value":             {Metric: "ifInErrors", Transform: TransformValue, Op: OpGT},
+		"rate on a gauge":              {Metric: "memTotalReal", Transform: TransformRate, Op: OpGT},
+		"percent_of without of":        {Metric: "hrProcessorLoad", Transform: TransformPercentOf, Op: OpGT},
+		"rate with max":                {Metric: "ifInErrors", Func: FuncMax, Op: OpGT},
+		"bad func":                     {Metric: "memTotalReal", Func: "p99", Op: OpGT},
+		"aggregate on interface":       {Metric: "ifOperStatus", Aggregate: AggMax, Op: OpEQ},
+		"bad aggregate":                {Metric: "hrProcessorLoad", Aggregate: "p95", Op: OpGT, Window: "10m"},
+		"window not a duration":        {Metric: "memTotalReal", Window: "5m] or vector(1) #", Op: OpGT},
+		"window too long":              {Metric: "memTotalReal", Window: "48h", Op: OpGT},
+		"hostile catalogue name":       {Metric: `evil"} or vector(1) or x{a="`, Op: OpGT},
+		"transform is not whitelisted": {Metric: "memTotalReal", Transform: `value"}`, Op: OpGT},
+	}
+	cases["window under twice interval"] = MetricSpec{Metric: "hrProcessorLoad", Window: "9m", Op: OpGT}
+	for name, spec := range cases {
+		t.Run(name, func(t *testing.T) {
+			if got, err := CompileMetric(&spec, cat, time.Minute); err == nil {
+				t.Fatalf("accepted, expr %q", got.Expr)
+			}
+		})
+	}
+}
+
+// TestHostileTextNeverReachesTheQuery proves the injection gate: the strings an admin controls (name, summary,
+// description, every inventory value) never change the generated expression, and a hostile metric name is not
+// in the catalogue at all.
+func TestHostileTextNeverReachesTheQuery(t *testing.T) {
+	cat := testCatalogue()
+	if _, ok := cat.Lookup(`evil"} or vector(1) or x{a="`); ok {
+		t.Fatal("a name that is not a plain identifier entered the catalogue")
+	}
+	base := Draft{Name: "r", Kind: KindMetric, Severity: "warning",
+		Spec: RuleSpec{Metric: &MetricSpec{Metric: "hrProcessorLoad", Aggregate: AggMax, Window: "10m", Op: OpGT, Threshold: 80}}}
+	want, err := Build(base, cat, time.Minute)
+	if err != nil {
+		t.Fatal(err)
+	}
+	hostile := []string{`"} or vector(1)`, "{{.Device}}", `x{a="b"}`, "`; DROP TABLE devices; --", "\"}[5m]) or vector(1) #", "{{"}
+	for _, h := range hostile {
+		d := base
+		d.Summary, d.Description = h, h
+		got, err := Build(d, cat, time.Minute)
+		if err == nil && got.Expr != want.Expr {
+			t.Errorf("text %q changed the expression: %q", h, got.Expr)
+		}
+		d.Name = h
+		if _, err := Build(d, cat, time.Minute); err == nil {
+			t.Errorf("hostile name %q accepted", h)
+		}
+	}
+	for _, h := range hostile {
+		s := base.Spec.Metric
+		s.Metric = h
+		if _, err := CompileMetric(s, cat, time.Minute); err == nil {
+			t.Errorf("hostile metric %q compiled", h)
+		}
+	}
+	// Whatever shape the expression has, it only contains the catalogue name and fixed template words.
+	for _, w := range []string{"vector", "DROP", `"}`} {
+		if strings.Contains(want.Expr, w) {
+			t.Errorf("expression %q contains %q", want.Expr, w)
+		}
+	}
+}
+
+func TestDefaultTextsRender(t *testing.T) {
+	cat := testCatalogue()
+	d := Draft{Name: "r", Kind: KindMetric, Severity: "info",
+		Spec: RuleSpec{Metric: &MetricSpec{Metric: "memTotalReal", Op: OpLT, Threshold: 1}}}
+	b, err := Build(d, cat, time.Minute)
+	if err != nil || b.Summary == "" || b.Description == "" {
+		t.Fatalf("defaults: %+v %v", b, err)
+	}
+	d.Summary = "{{.NoSuchField}}"
+	if _, err := Build(d, cat, time.Minute); err == nil {
+		t.Fatal("template with an unknown field accepted")
+	}
+}

internal/alertrule/spec.go

@@ -0,0 +1,90 @@
+// Package alertrule is the builder behind custom alert rules. An admin composes a rule from whitelisted parts
+// (a catalogue metric, or device and interface fields); this package validates the composition and turns it
+// into a MetricsQL expression (metric rules) or a Go predicate over a row (inventory rules). It is pure: no
+// store, no clock, no HTTP. Nothing a user types is ever pasted into a query: metric names are looked up in the
+// catalogue, durations and numbers are parsed and re-printed, and every other string only feeds text templates.
+package alertrule
+
+import "time"
+
+// Kinds of custom rule.
+const (
+	KindMetric    = "metric"
+	KindInventory = "inventory"
+)
+
+// Transforms of a metric rule: how the stored series becomes the measured value.
+const (
+	TransformValue     = "value"
+	TransformRate      = "rate"
+	TransformPercentOf = "percent_of"
+)
+
+// Funcs reduce the samples of the window to one value.
+const (
+	FuncLast = "last"
+	FuncAvg  = "avg"
+	FuncMax  = "max"
+	FuncMin  = "min"
+)
+
+// Aggregates collapse the rows of a component-level metric into one value per device.
+const (
+	AggEach = "each"
+	AggMax  = "max"
+	AggMin  = "min"
+	AggAvg  = "avg"
+	AggSum  = "sum"
+)
+
+// Comparison operators of a metric rule (the evaluator compares the measured value with the threshold).
+const (
+	OpGT = ">"
+	OpGE = ">="
+	OpLT = "<"
+	OpLE = "<="
+	OpEQ = "=="
+	OpNE = "!="
+)
+
+// Ops lists every comparison operator.
+var Ops = []string{OpGT, OpGE, OpLT, OpLE, OpEQ, OpNE}
+
+// Window bounds of a metric rule. The lower bound is also 2x the metric's poll interval.
+const (
+	DefaultWindow = 5 * time.Minute
+	MaxWindow     = 24 * time.Hour
+)
+
+// MaxConditions is how many conditions an inventory rule may have.
+const MaxConditions = 5
+
+// RuleSpec is the builder input of a custom rule. Exactly the member that matches the rule's kind is set.
+type RuleSpec struct {
+	Metric    *MetricSpec    `json:"metric,omitempty"`
+	Inventory *InventorySpec `json:"inventory,omitempty"`
+}
+
+// MetricSpec composes a MetricsQL rule from one catalogue metric.
+type MetricSpec struct {
+	Metric    string  `json:"metric" minLength:"1" doc:"A metric name from GET /alert-rules/catalogue."`
+	Transform string  `json:"transform,omitempty" enum:"value,rate,percent_of" doc:"Default value; rate for counters; percent_of needs an of."`
+	Window    string  `json:"window,omitempty" doc:"A duration such as 5m (default); at least twice the poll interval, at most 24h."`
+	Func      string  `json:"func,omitempty" enum:"last,avg,max,min" doc:"Reduces the window to one value; default last. A rate averages."`
+	Aggregate string  `json:"aggregate,omitempty" enum:"each,max,min,avg,sum" doc:"Component metrics: each row (default) or per device."`
+	Op        string  `json:"op" enum:">,>=,<,<=,==,!="`
+	Threshold float64 `json:"threshold"`
+}
+
+// InventorySpec composes a rule over device or interface rows.
+type InventorySpec struct {
+	Conditions []InventoryCondition `json:"conditions" minItems:"1" maxItems:"5" doc:"ANDed. All on device fields or all on interface fields."`
+}
+
+// InventoryCondition tests one field. Value is text for every type: a number, a duration such as 1h, a string or a
+// regular expression; it is unused by the bool ops and by empty and not_empty.
+type InventoryCondition struct {
+	Field string `json:"field" minLength:"1" doc:"A field name from GET /alert-rules/catalogue."`
+	Op    string `json:"op" minLength:"1" doc:"An op the field's type allows; see the catalogue."`
+	Value string `json:"value,omitempty"`
+}

internal/alertrule/validate.go

@@ -0,0 +1,131 @@
+package alertrule
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+	"text/template"
+	"time"
+)
+
+// Draft is a custom rule as an admin writes it, before it is stored.
+type Draft struct {
+	Name        string
+	Kind        string
+	Severity    string
+	For         time.Duration
+	Summary     string
+	Description string
+	Spec        RuleSpec
+}
+
+// Built is a validated draft: the compiled spec plus the texts the evaluator will render.
+type Built struct {
+	Compiled
+	// Inventory is set for an inventory rule.
+	Inventory *InventoryRule
+	// Summary and Description are the draft's, or a generated default when the draft left them empty.
+	Summary, Description string
+}
+
+// TemplateData mirrors alert.TemplateData, the fields a summary or description may use. A test in
+// internal/app keeps the two in step; it exists so a template naming a missing field is refused at save time.
+type TemplateData struct {
+	Rule, Severity, Device, Iface, Component string
+	Value, Threshold                         float64
+	Op                                       string
+	For                                      time.Duration
+}
+
+var nameRE = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$`)
+
+// Build validates d against the catalogue. All problems are reported together.
+func Build(d Draft, cat Catalogue, fallbackInterval time.Duration) (Built, error) {
+	var errs []error
+	if !nameRE.MatchString(d.Name) {
+		errs = append(errs, errors.New("name must be 1 to 64 letters, digits, '_', '-' or '.', starting with a letter or digit"))
+	}
+	switch d.Severity {
+	case "critical", "warning", "info":
+	default:
+		errs = append(errs, fmt.Errorf("severity %q is not critical, warning or info", d.Severity))
+	}
+	if d.For < 0 {
+		errs = append(errs, errors.New("for is negative"))
+	}
+	b, err := buildSpec(d, cat, fallbackInterval)
+	if err != nil {
+		errs = append(errs, err)
+	}
+	b.Summary, b.Description = d.Summary, d.Description
+	if strings.TrimSpace(b.Summary) == "" {
+		b.Summary = defaultSummary(d.Kind, b.Level)
+	}
+	if strings.TrimSpace(b.Description) == "" {
+		b.Description = defaultDescription(d.Kind)
+	}
+	errs = append(errs, checkTemplate("summary", b.Summary, d), checkTemplate("description", b.Description, d))
+	if err := errors.Join(errs...); err != nil {
+		return Built{}, err
+	}
+	return b, nil
+}
+
+func buildSpec(d Draft, cat Catalogue, fallbackInterval time.Duration) (Built, error) {
+	switch d.Kind {
+	case KindMetric:
+		if d.Spec.Inventory != nil {
+			return Built{}, errors.New("a metric rule takes spec.metric, not spec.inventory")
+		}
+		c, err := CompileMetric(d.Spec.Metric, cat, fallbackInterval)
+		return Built{Compiled: c}, err
+	case KindInventory:
+		if d.Spec.Metric != nil {
+			return Built{}, errors.New("an inventory rule takes spec.inventory, not spec.metric")
+		}
+		r, err := CompileInventory(d.Spec.Inventory)
+		if err != nil {
+			return Built{}, err
+		}
+		return Built{Compiled: Compiled{Level: r.Entity, Labels: labelsOf(r.Entity), Op: OpGT, Threshold: 0}, Inventory: r}, nil
+	}
+	return Built{}, fmt.Errorf("kind %q is not metric or inventory", d.Kind)
+}
+
+// checkTemplate parses text and runs it once against sample data, so unknown fields fail at save time.
+func checkTemplate(name, text string, d Draft) error {
+	t, err := template.New(name).Option("missingkey=error").Parse(text)
+	if err != nil {
+		return fmt.Errorf("%s template: %w", name, err)
+	}
+	var sb strings.Builder
+	sample := TemplateData{Rule: d.Name, Severity: d.Severity, Device: "device", Iface: "iface", Component: "component", Op: OpGT, For: d.For}
+	if err := t.Execute(&sb, sample); err != nil {
+		return fmt.Errorf("%s template: %w", name, err)
+	}
+	return nil
+}
+
+// defaultSummary names the entity and, for a metric rule, the value against the threshold. It never embeds
+// catalogue text, only template actions, so a default cannot carry markup of its own.
+func defaultSummary(kind, level string) string {
+	who := "{{.Device}}"
+	switch level {
+	case LevelInterface:
+		who += " {{.Iface}}"
+	case LevelComponent:
+		who += " {{.Component}}"
+	}
+	if kind == KindInventory {
+		return who + ` matches rule {{.Rule}}`
+	}
+	return who + ` {{.Rule}}: {{printf "%.4g" .Value}} {{.Op}} {{printf "%.4g" .Threshold}}`
+}
+
+func defaultDescription(kind string) string {
+	if kind == KindInventory {
+		return `{{.Device}} meets every condition of rule {{.Rule}}.`
+	}
+	return `Rule {{.Rule}} measured {{printf "%.4g" .Value}} on {{.Device}} (alerts when {{.Op}} {{printf "%.4g" .Threshold}}, for {{.For}}).`
+}

internal/api/alert_policies.go

@@ -11,16 +11,27 @@ import (
 	"github.com/danielgtaylor/huma/v2"
 
 	"github.com/v10networks/nmon/internal/alertpolicy"
+	"github.com/v10networks/nmon/internal/alertrule"
 	"github.com/v10networks/nmon/internal/store"
 )
 
-// AlertRuleInfo describes a rule a policy can bind, for the UI's dropdown.
+// AlertRuleInfo describes a rule a policy can bind, for the UI's dropdown and the Alert rules tab.
 type AlertRuleInfo struct {
-	Name      string   `json:"name"`
-	Level     string   `json:"level" enum:"device,interface,component,system" doc:"What the rule's entity is; port matchers need interface."`
-	Op        string   `json:"op,omitempty" enum:">,<" doc:"Comparison of a metric rule; empty for reachability and system rules."`
-	Threshold *float64 `json:"threshold" nullable:"true" doc:"The rule's default threshold; null for reachability and system rules."`
-	Severity  string   `json:"severity" enum:"critical,warning,info"`
+	ID          string              `json:"id" doc:"A custom rule's ULID; for every other kind the rule name."`
+	Name        string              `json:"name"`
+	Kind        string              `json:"kind" enum:"builtin,config,metric,inventory" doc:"builtin ships with nmon; config is deprecated."`
+	Editable    bool                `json:"editable" doc:"True for custom rules only."`
+	Enabled     bool                `json:"enabled"`
+	Level       string              `json:"level" enum:"device,interface,component,system" doc:"What the rule's entity is."`
+	Op          string              `json:"op,omitempty" enum:">,>=,<,<=,==,!=" doc:"Comparison of a metric rule; empty if none."`
+	Threshold   *float64            `json:"threshold" nullable:"true" doc:"The rule's default threshold; null if it has none."`
+	Severity    string              `json:"severity" enum:"critical,warning,info"`
+	For         *string             `json:"for" nullable:"true" doc:"How long the condition must hold, a duration such as 5m."`
+	Summary     string              `json:"summary,omitempty" doc:"The summary template."`
+	Spec        *alertrule.RuleSpec `json:"spec,omitempty" doc:"The builder spec of a custom rule."`
+	Expr        string              `json:"expr,omitempty" doc:"The MetricsQL a rule runs, read-only; empty if it runs none."`
+	PolicyCount int                 `json:"policy_count" doc:"How many alert policies name this rule."`
+	Copyable    bool                `json:"copyable" doc:"True for a built-in the builder can express: POST /alert-rules/{id}/copy clones it."`
 }
 
 // AlertRuleList is every rule a policy can bind.
@@ -137,12 +148,12 @@ func (s *server) policyHost() (AlertPolicyHost, error) {
 	return s.AlertPolicy, nil
 }
 
-func (s *server) listAlertRules(_ context.Context, _ *struct{}) (*struct{ Body AlertRuleList }, error) {
-	h, err := s.policyHost()
+func (s *server) listAlertRules(ctx context.Context, _ *struct{}) (*struct{ Body AlertRuleList }, error) {
+	items, err := s.ruleInfos(ctx)
 	if err != nil {
 		return nil, err
 	}
-	return &struct{ Body AlertRuleList }{AlertRuleList{Items: h.Rules()}}, nil
+	return &struct{ Body AlertRuleList }{AlertRuleList{Items: items}}, nil
 }
 
 func policyView(r store.AlertPolicy) (AlertPolicyView, error) {

internal/api/alert_rules.go

@@ -0,0 +1,456 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/alertrule"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// AlertRuleHost is the rule registry as the API sees it: the catalogue of what a builder may use, validation
+// and a dry run of a draft, and the combined rule list.
+type AlertRuleHost interface {
+	// Rules lists every rule: built-in, config and custom (and the reachability and system rules).
+	Rules() []AlertRuleInfo
+	// Catalogue lists the metrics and inventory fields a builder may use.
+	Catalogue(ctx context.Context) RuleCatalogue
+	// Build validates a draft; the error text lists every problem.
+	Build(ctx context.Context, d alertrule.Draft) (RuleBuilt, error)
+	// Test compiles a draft and runs it once against the live data.
+	Test(ctx context.Context, d alertrule.Draft) (RuleTestResult, error)
+	// Reserved reports a name a custom rule cannot take: any built-in, config, reachability or system rule.
+	Reserved(name string) bool
+	// Copy returns the builder draft equivalent to a built-in rule. A non-empty reason says why it cannot.
+	Copy(ctx context.Context, name string) (d alertrule.Draft, reason string)
+	// Changed drops the registry's caches so a write applies on the next evaluation cycle.
+	Changed()
+}
+
+// RuleBuilt is what a valid draft compiles to.
+type RuleBuilt struct {
+	Expr  string
+	Level string
+}
+
+// RuleCatalogue is what the builder offers.
+type RuleCatalogue struct {
+	Metrics []alertrule.MetricInfo     `json:"metrics"`
+	Fields  []alertrule.InventoryField `json:"fields" doc:"Inventory fields with the ops each type accepts."`
+	Ops     []string                   `json:"ops" doc:"Comparison operators of a metric rule."`
+	Window  RuleWindowLimits           `json:"window"`
+}
+
+// RuleWindowLimits are the bounds of a metric rule's window.
+type RuleWindowLimits struct {
+	DefaultSeconds int64 `json:"default_seconds"`
+	MaxSeconds     int64 `json:"max_seconds"`
+}
+
+// AlertRuleBody is the body of POST and PUT /alert-rules and of POST /alert-rules/test.
+type AlertRuleBody struct {
+	Name        string             `json:"name" minLength:"1" maxLength:"64" doc:"Unique across all rules. Policies name a rule by this."`
+	Kind        string             `json:"kind" enum:"metric,inventory"`
+	Enabled     *bool              `json:"enabled,omitempty" doc:"Omitted: on for create, kept for update. Off retires open alerts silently."`
+	Severity    string             `json:"severity" enum:"critical,warning,info"`
+	For         *string            `json:"for,omitempty" doc:"How long the condition must hold, a duration such as 15m. Default 0."`
+	Summary     string             `json:"summary,omitempty" doc:"A text/template over rule, device, iface, component, value, threshold, op."`
+	Description string             `json:"description,omitempty" doc:"As summary."`
+	Spec        alertrule.RuleSpec `json:"spec"`
+}
+
+// AlertRuleCopyBody is the optional body of POST /alert-rules/{id}/copy.
+type AlertRuleCopyBody struct {
+	Name string `json:"name,omitempty" maxLength:"64" doc:"Name of the copy; default is the built-in's name plus _custom."`
+}
+
+// RuleTestResult is a dry run of a draft.
+type RuleTestResult struct {
+	Expr      string           `json:"expr" doc:"The generated query; empty for an inventory rule."`
+	Level     string           `json:"level" enum:"device,interface,component"`
+	Entities  int              `json:"entities" doc:"Series (metric rule) or rows (inventory rule) the rule saw."`
+	Breaching int              `json:"breaching" doc:"How many breach right now, before any alert policy is applied."`
+	Samples   []RuleTestSample `json:"samples" doc:"Up to 20, breaching ones first."`
+}
+
+// RuleTestSample is one entity of a dry run.
+type RuleTestSample struct {
+	Device    string  `json:"device" doc:"Device slug."`
+	Iface     string  `json:"iface,omitempty" doc:"Interface ULID."`
+	Component string  `json:"component,omitempty"`
+	Value     float64 `json:"value"`
+	Breach    bool    `json:"breach"`
+}
+
+func (s *server) registerAlertRules(api huma.API) {
+	tags := []string{"alerts"}
+	bad := []int{http.StatusConflict, http.StatusUnprocessableEntity, http.StatusServiceUnavailable}
+	huma.Register(api, huma.Operation{OperationID: "get-alert-rule-catalogue", Method: http.MethodGet, Path: "/api/v1/alert-rules/catalogue",
+		Tags: tags, Summary: "Metrics and inventory fields the builder offers", Errors: []int{http.StatusServiceUnavailable}}, s.ruleCatalogue)
+	huma.Register(api, huma.Operation{OperationID: "test-alert-rule", Method: http.MethodPost, Path: "/api/v1/alert-rules/test", Tags: tags,
+		Summary:     "Compile a draft rule and run it once",
+		Description: "Returns the generated query, how many entities the rule sees and how many breach now, ignoring alert policies.",
+		Errors:      []int{http.StatusUnprocessableEntity, http.StatusServiceUnavailable}}, s.testAlertRule)
+	huma.Register(api, huma.Operation{OperationID: "create-alert-rule", Method: http.MethodPost, Path: "/api/v1/alert-rules", Tags: tags,
+		DefaultStatus: http.StatusCreated, Summary: "Create a custom alert rule",
+		Description: "Builder only: a metric rule names a catalogue metric, an inventory rule names whitelisted fields. " +
+			"A rule alerts only where an alert policy binds it.", Errors: bad}, s.createAlertRule)
+	huma.Register(api, huma.Operation{OperationID: "get-alert-rule", Method: http.MethodGet, Path: "/api/v1/alert-rules/{id}", Tags: tags,
+		Summary: "An alert rule", Errors: []int{http.StatusNotFound, http.StatusServiceUnavailable}}, s.getAlertRule)
+	huma.Register(api, huma.Operation{OperationID: "update-alert-rule", Method: http.MethodPut, Path: "/api/v1/alert-rules/{id}", Tags: tags,
+		Summary: "Replace a custom alert rule", Description: "A built-in or config rule is 409.",
+		Errors: append([]int{http.StatusNotFound}, bad...)}, s.updateAlertRule)
+	huma.Register(api, huma.Operation{OperationID: "delete-alert-rule", Method: http.MethodDelete, Path: "/api/v1/alert-rules/{id}",
+		Tags:          tags,
+		DefaultStatus: http.StatusNoContent, Summary: "Delete a custom alert rule",
+		Description: "409 while an alert policy names the rule, or for a built-in. Open alerts of the rule close silently as retired.",
+		Errors:      []int{http.StatusNotFound, http.StatusConflict, http.StatusServiceUnavailable}}, s.deleteAlertRule)
+	huma.Register(api, huma.Operation{OperationID: "copy-alert-rule", Method: http.MethodPost, Path: "/api/v1/alert-rules/{id}/copy",
+		Tags:          tags,
+		DefaultStatus: http.StatusCreated, Summary: "Clone a built-in rule into a custom rule",
+		Description: "422 with the reason when the builder cannot express the built-in.",
+		Errors:      append([]int{http.StatusNotFound}, bad...)}, s.copyAlertRule)
+}
+
+func (s *server) ruleHost() (AlertRuleHost, error) {
+	if s.AlertRules == nil {
+		return nil, errNotConfigured("alert rules")
+	}
+	return s.AlertRules, nil
+}
+
+// ruleInfos is the rule list with each rule's policy count.
+func (s *server) ruleInfos(ctx context.Context) ([]AlertRuleInfo, error) {
+	var items []AlertRuleInfo
+	if s.AlertRules != nil {
+		items = s.AlertRules.Rules()
+	} else {
+		h, err := s.policyHost()
+		if err != nil {
+			return nil, err
+		}
+		items = h.Rules()
+	}
+	pols, err := s.Store.ListAlertPolicies(ctx)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	counts := map[string]int{}
+	for _, p := range pols {
+		counts[p.Rule]++
+	}
+	for i := range items {
+		items[i].PolicyCount = counts[items[i].Name]
+	}
+	return items, nil
+}
+
+func (s *server) ruleInfo(ctx context.Context, id string) (AlertRuleInfo, error) {
+	items, err := s.ruleInfos(ctx)
+	if err != nil {
+		return AlertRuleInfo{}, err
+	}
+	for _, it := range items {
+		if it.ID == id || (it.Kind != "metric" && it.Kind != "inventory" && it.Name == id) {
+			return it, nil
+		}
+	}
+	return AlertRuleInfo{}, huma.Error404NotFound("not found")
+}
+
+func (s *server) getAlertRule(ctx context.Context, in *struct {
+	ID string `path:"id" doc:"A custom rule's ULID, or a built-in's name."`
+}) (*struct{ Body AlertRuleInfo }, error) {
+	if _, err := s.ruleHost(); err != nil {
+		return nil, err
+	}
+	it, err := s.ruleInfo(ctx, in.ID)
+	if err != nil {
+		return nil, err
+	}
+	return &struct{ Body AlertRuleInfo }{it}, nil
+}
+
+func (s *server) ruleCatalogue(ctx context.Context, _ *struct{}) (*struct{ Body RuleCatalogue }, error) {
+	h, err := s.ruleHost()
+	if err != nil {
+		return nil, err
+	}
+	return &struct{ Body RuleCatalogue }{h.Catalogue(ctx)}, nil
+}
+
+// draftOf converts a request body to a builder draft.
+func draftOf(b AlertRuleBody) (alertrule.Draft, error) {
+	d := alertrule.Draft{Name: strings.TrimSpace(b.Name), Kind: b.Kind, Severity: b.Severity, Summary: b.Summary,
+		Description: b.Description, Spec: b.Spec}
+	if b.For != nil {
+		f, err := time.ParseDuration(*b.For)
+		if err != nil {
+			return d, huma.Error422UnprocessableEntity("for: " + err.Error())
+		}
+		d.For = f
+	}
+	return d, nil
+}
+
+func (s *server) testAlertRule(ctx context.Context, in *struct{ Body AlertRuleBody }) (*struct{ Body RuleTestResult }, error) {
+	h, err := s.ruleHost()
+	if err != nil {
+		return nil, err
+	}
+	d, err := draftOf(in.Body)
+	if err != nil {
+		return nil, err
+	}
+	res, err := h.Test(ctx, d)
+	if err != nil {
+		return nil, s.ruleErr(err)
+	}
+	return &struct{ Body RuleTestResult }{res}, nil
+}
+
+// ruleErr maps a registry error: a validation problem is 422, anything else is logged and 500.
+func (s *server) ruleErr(err error) error {
+	var ve *RuleInvalid
+	if errors.As(err, &ve) {
+		return huma.Error422UnprocessableEntity(flatten(ve))
+	}
+	s.Logger.Error("alert rule", "err", err)
+	return huma.Error500InternalServerError("internal error")
+}
+
+// RuleInvalid is a draft that fails validation.
+type RuleInvalid struct{ Err error }
+
+func (e *RuleInvalid) Error() string { return e.Err.Error() }
+func (e *RuleInvalid) Unwrap() error { return e.Err }
+
+// ruleRow validates a body and converts it to a store row; enabled is the value to keep when the body omits it.
+func (s *server) ruleRow(ctx context.Context, h AlertRuleHost, b AlertRuleBody, enabled bool) (store.AlertRule, RuleBuilt, error) {
+	d, err := draftOf(b)
+	if err != nil {
+		return store.AlertRule{}, RuleBuilt{}, err
+	}
+	built, err := h.Build(ctx, d)
+	if err != nil {
+		return store.AlertRule{}, RuleBuilt{}, s.ruleErr(err)
+	}
+	spec, err := json.Marshal(d.Spec)
+	if err != nil {
+		return store.AlertRule{}, RuleBuilt{}, huma.Error500InternalServerError("internal error")
+	}
+	if b.Enabled != nil {
+		enabled = *b.Enabled
+	}
+	return store.AlertRule{Name: d.Name, Kind: d.Kind, Enabled: enabled, Severity: d.Severity, ForSeconds: int64(d.For.Seconds()),
+		Summary: d.Summary, Description: d.Description, Spec: string(spec)}, built, nil
+}
+
+func (s *server) createAlertRule(ctx context.Context, in *struct{ Body AlertRuleBody }) (*struct{ Body AlertRuleInfo }, error) {
+	h, err := s.ruleHost()
+	if err != nil {
+		return nil, err
+	}
+	return s.storeNewRule(ctx, h, in.Body, "alert_rule.create")
+}
+
+func (s *server) storeNewRule(ctx context.Context, h AlertRuleHost, b AlertRuleBody, action string) (*struct{ Body AlertRuleInfo }, error) {
+	if h.Reserved(strings.TrimSpace(b.Name)) {
+		return nil, huma.Error409Conflict(fmt.Sprintf("rule name %q is taken by a built-in, config or reachability rule", b.Name))
+	}
+	row, _, err := s.ruleRow(ctx, h, b, true)
+	if err != nil {
+		return nil, err
+	}
+	err = s.Store.WithTx(ctx, func(tx store.Store) error {
+		if err := tx.CreateAlertRule(ctx, &row); err != nil {
+			return err
+		}
+		return tx.AppendAudit(ctx, actorOf(ctx), action, row.ID, nil, b)
+	})
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	h.Changed()
+	return s.ruleResult(ctx, row.ID)
+}
+
+func (s *server) ruleResult(ctx context.Context, id string) (*struct{ Body AlertRuleInfo }, error) {
+	it, err := s.ruleInfo(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+	return &struct{ Body AlertRuleInfo }{it}, nil
+}
+
+// customRule loads a custom rule, or says why the id cannot be edited.
+func (s *server) customRule(ctx context.Context, tx store.AlertRules, id string) (*store.AlertRule, error) {
+	r, err := tx.GetAlertRule(ctx, id)
+	if err == nil {
+		return r, nil
+	}
+	if errors.Is(err, store.ErrNotFound) {
+		if it, ierr := s.ruleInfo(ctx, id); ierr == nil && !it.Editable {
+			return nil, huma.Error409Conflict(fmt.Sprintf("rule %q is %s and cannot be changed; copy it to a custom rule", it.Name, it.Kind))
+		}
+	}
+	return nil, s.storeErr(err)
+}
+
+// referencing lists the names of policies that bind rule.
+func referencing(ctx context.Context, tx store.AlertPolicies, rule string) ([]string, error) {
+	ps, err := tx.ListAlertPolicies(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var names []string
+	for _, p := range ps {
+		if p.Rule == rule {
+			names = append(names, p.Name)
+		}
+	}
+	return names, nil
+}
+
+func (s *server) updateAlertRule(ctx context.Context, in *struct {
+	ID   string `path:"id" doc:"A custom rule's ULID."`
+	Body AlertRuleBody
+}) (*struct{ Body AlertRuleInfo }, error) {
+	h, err := s.ruleHost()
+	if err != nil {
+		return nil, err
+	}
+	cur, err := s.customRule(ctx, s.Store, in.ID)
+	if err != nil {
+		return nil, err
+	}
+	name := strings.TrimSpace(in.Body.Name)
+	if name != cur.Name && h.Reserved(name) {
+		return nil, huma.Error409Conflict(fmt.Sprintf("rule name %q is taken by a built-in, config or reachability rule", name))
+	}
+	row, built, err := s.ruleRow(ctx, h, in.Body, cur.Enabled)
+	if err != nil {
+		return nil, err
+	}
+	row.ID, row.CreatedAt = cur.ID, cur.CreatedAt
+	oldLevel := "" // read before the transaction: a second store call inside it would wait for the first forever on SQLite
+	if old, ierr := s.ruleInfo(ctx, cur.ID); ierr == nil {
+		oldLevel = old.Level
+	}
+	err = s.Store.WithTx(ctx, func(tx store.Store) error {
+		if err := checkEditable(ctx, tx, cur, row.Name, oldLevel, built); err != nil {
+			return err
+		}
+		if err := tx.UpdateAlertRule(ctx, &row); err != nil {
+			return err
+		}
+		return tx.AppendAudit(ctx, actorOf(ctx), "alert_rule.update", row.ID, cur, in.Body)
+	})
+	if err != nil {
+		return nil, s.ruleStoreErr(err)
+	}
+	h.Changed()
+	return s.ruleResult(ctx, row.ID)
+}
+
+// checkEditable refuses a rename, or a change of level, while policies name the rule: both would orphan or
+// invalidate them (a policy with port matchers needs an interface rule).
+func checkEditable(ctx context.Context, tx store.AlertPolicies, cur *store.AlertRule, newName, oldLevel string, built RuleBuilt) error {
+	names, err := referencing(ctx, tx, cur.Name)
+	if err != nil || len(names) == 0 {
+		return err
+	}
+	if newName != cur.Name {
+		return &ruleBusy{fmt.Sprintf("policies %s name this rule; they would lose it on a rename", strings.Join(names, ", "))}
+	}
+	if oldLevel != "" && oldLevel != built.Level {
+		return &ruleBusy{fmt.Sprintf("policies %s bind this rule at the %s level; change them first", strings.Join(names, ", "), oldLevel)}
+	}
+	return nil
+}
+
+// ruleBusy is a write the rule's policies forbid.
+type ruleBusy struct{ msg string }
+
+func (e *ruleBusy) Error() string { return e.msg }
+
+func (s *server) ruleStoreErr(err error) error {
+	var rb *ruleBusy
+	if errors.As(err, &rb) {
+		return huma.Error409Conflict(rb.msg)
+	}
+	return s.storeErr(err)
+}
+
+func (s *server) deleteAlertRule(ctx context.Context, in *struct {
+	ID string `path:"id" doc:"A custom rule's ULID."`
+}) (*struct{}, error) {
+	h, err := s.ruleHost()
+	if err != nil {
+		return nil, err
+	}
+	if _, err := s.customRule(ctx, s.Store, in.ID); err != nil {
+		return nil, err
+	}
+	err = s.Store.WithTx(ctx, func(tx store.Store) error {
+		cur, err := tx.GetAlertRule(ctx, in.ID)
+		if err != nil {
+			return err
+		}
+		names, err := referencing(ctx, tx, cur.Name)
+		if err != nil {
+			return err
+		}
+		if len(names) > 0 {
+			return &ruleBusy{fmt.Sprintf("alert policies %s still bind rule %q; delete or change them first", strings.Join(names, ", "), cur.Name)}
+		}
+		if err := tx.DeleteAlertRule(ctx, in.ID); err != nil {
+			return err
+		}
+		return tx.AppendAudit(ctx, actorOf(ctx), "alert_rule.delete", in.ID, cur, nil)
+	})
+	if err != nil {
+		return nil, s.ruleStoreErr(err)
+	}
+	h.Changed()
+	return nil, nil
+}
+
+func (s *server) copyAlertRule(ctx context.Context, in *struct {
+	ID   string `path:"id" doc:"A built-in rule's name."`
+	Body AlertRuleCopyBody
+}) (*struct{ Body AlertRuleInfo }, error) {
+	h, err := s.ruleHost()
+	if err != nil {
+		return nil, err
+	}
+	src, err := s.ruleInfo(ctx, in.ID)
+	if err != nil {
+		return nil, err
+	}
+	if src.Editable {
+		return nil, huma.Error409Conflict("only a built-in rule can be copied; this one is already custom")
+	}
+	d, reason := h.Copy(ctx, src.Name)
+	if reason != "" {
+		return nil, huma.Error422UnprocessableEntity(fmt.Sprintf("%s cannot be copied to the builder: %s", src.Name, reason))
+	}
+	if in.Body.Name != "" {
+		d.Name = in.Body.Name
+	}
+	return s.storeNewRule(ctx, h, bodyOf(d), "alert_rule.copy")
+}
+
+// bodyOf converts a draft back to a request body.
+func bodyOf(d alertrule.Draft) AlertRuleBody {
+	f := d.For.String()
+	return AlertRuleBody{Name: d.Name, Kind: d.Kind, Severity: d.Severity, For: &f, Summary: d.Summary,
+		Description: d.Description, Spec: d.Spec}
+}

internal/api/alert_rules_test.go

@@ -0,0 +1,248 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/alertrule"
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// fakeRuleHost stands in for the registry: one built-in plus the custom rules in the store.
+type fakeRuleHost struct {
+	st      store.Store
+	changes int
+	cat     alertrule.Catalogue
+}
+
+func newFakeRuleHost() *fakeRuleHost {
+	return &fakeRuleHost{cat: alertrule.NewCatalogue([]alertrule.MetricInfo{
+		{Name: "hrProcessorLoad", Type: "gauge", Level: alertrule.LevelComponent, IntervalSeconds: 60},
+		{Name: "memTotalReal", Type: "gauge", Level: alertrule.LevelDevice, IntervalSeconds: 60},
+	})}
+}
+
+func (f *fakeRuleHost) Rules() []api.AlertRuleInfo {
+	thr := 90.0
+	out := []api.AlertRuleInfo{{ID: "cpu_high", Name: "cpu_high", Kind: "builtin", Enabled: true, Level: "device", Op: ">", Threshold: &thr,
+		Severity: "warning", Copyable: true}}
+	rows, _ := f.st.ListAlertRules(context.Background())
+	for _, r := range rows {
+		var sp alertrule.RuleSpec
+		_ = json.Unmarshal([]byte(r.Spec), &sp)
+		level := "device"
+		if r.Kind == "inventory" && strings.Contains(r.Spec, "if_") {
+			level = "interface"
+		}
+		out = append(out, api.AlertRuleInfo{ID: r.ID, Name: r.Name, Kind: r.Kind, Editable: true, Enabled: r.Enabled, Level: level,
+			Severity: r.Severity, Spec: &sp})
+	}
+	return out
+}
+
+func (f *fakeRuleHost) Catalogue(context.Context) api.RuleCatalogue {
+	return api.RuleCatalogue{Metrics: f.cat.List(), Fields: alertrule.Fields(), Ops: alertrule.Ops}
+}
+
+func (f *fakeRuleHost) Build(_ context.Context, d alertrule.Draft) (api.RuleBuilt, error) {
+	b, err := alertrule.Build(d, f.cat, time.Minute)
+	if err != nil {
+		return api.RuleBuilt{}, &api.RuleInvalid{Err: err}
+	}
+	return api.RuleBuilt{Expr: b.Expr, Level: b.Level}, nil
+}
+
+func (f *fakeRuleHost) Test(ctx context.Context, d alertrule.Draft) (api.RuleTestResult, error) {
+	b, err := f.Build(ctx, d)
+	if err != nil {
+		return api.RuleTestResult{}, err
+	}
+	return api.RuleTestResult{Expr: b.Expr, Level: b.Level, Entities: 3, Breaching: 1,
+		Samples: []api.RuleTestSample{{Device: "d1", Value: 95, Breach: true}}}, nil
+}
+
+func (f *fakeRuleHost) Reserved(name string) bool { return name == "cpu_high" || name == "device_down" }
+
+func (f *fakeRuleHost) Copy(_ context.Context, name string) (alertrule.Draft, string) {
+	if name != "cpu_high" {
+		return alertrule.Draft{}, "it divides two metrics"
+	}
+	return alertrule.Draft{Name: "cpu_high_custom", Kind: alertrule.KindMetric, Severity: "warning", For: 10 * time.Minute,
+		Spec: alertrule.RuleSpec{Metric: &alertrule.MetricSpec{Metric: "hrProcessorLoad", Aggregate: "avg", Window: "15m", Op: ">", Threshold: 90}}}, ""
+}
+
+func (f *fakeRuleHost) Changed() { f.changes++ }
+
+func ruleEnv(t *testing.T) (*env, *fakeRuleHost) {
+	h, ph := newFakeRuleHost(), &fakePolicyHost{}
+	e := newEnvDeps(t, 0, func(d *api.Deps) { d.AlertRules, d.AlertPolicy = h, ph })
+	h.st = e.st
+	return e, h
+}
+
+const cpuRule = `{"name":"cpu_server","kind":"metric","severity":"warning","for":"15m",
+"spec":{"metric":{"metric":"hrProcessorLoad","aggregate":"max","window":"10m","op":">=","threshold":80}}}`
+
+func createRule(t *testing.T, e *env, body string) api.AlertRuleInfo {
+	t.Helper()
+	rec := e.do("POST", "/api/v1/alert-rules", body)
+	if rec.Code != http.StatusCreated {
+		t.Fatalf("POST: %d %s", rec.Code, rec.Body)
+	}
+	var v api.AlertRuleInfo
+	if err := json.Unmarshal(rec.Body.Bytes(), &v); err != nil || v.ID == "" {
+		t.Fatalf("created = %s (%v)", rec.Body, err)
+	}
+	return v
+}
+
+func TestCustomRuleCRUDIsAuditedAndTellsTheRegistry(t *testing.T) {
+	e, host := ruleEnv(t)
+	v := createRule(t, e, cpuRule)
+	if v.Kind != "metric" || !v.Editable || v.PolicyCount != 0 {
+		t.Fatalf("created = %+v", v)
+	}
+	isProblem(t, e.do("POST", "/api/v1/alert-rules", cpuRule), http.StatusConflict)
+	upd := strings.Replace(cpuRule, `"threshold":80`, `"threshold":85`, 1)
+	if rec := e.do("PUT", "/api/v1/alert-rules/"+v.ID, upd); rec.Code != http.StatusOK {
+		t.Fatalf("PUT: %d %s", rec.Code, rec.Body)
+	}
+	var got api.AlertRuleInfo
+	e.getJSON(t, "/api/v1/alert-rules/"+v.ID, &got)
+	if got.Spec == nil || got.Spec.Metric.Threshold != 85 {
+		t.Fatalf("after PUT: %+v", got)
+	}
+	row, _ := e.st.GetAlertRule(t.Context(), v.ID)
+	if row.ForSeconds != 900 || !row.Enabled {
+		t.Fatalf("row = %+v", row)
+	}
+	if rec := e.do("DELETE", "/api/v1/alert-rules/"+v.ID, ""); rec.Code != http.StatusNoContent {
+		t.Fatalf("DELETE: %d %s", rec.Code, rec.Body)
+	}
+	isProblem(t, e.do("GET", "/api/v1/alert-rules/"+v.ID, ""), http.StatusNotFound)
+	if host.changes != 3 {
+		t.Errorf("registry told %d times, want 3", host.changes)
+	}
+	audit, _, _ := e.st.ListAudit(t.Context(), store.Page{})
+	if len(audit) < 3 || audit[0].Action != "alert_rule.delete" {
+		t.Fatalf("audit = %+v", audit)
+	}
+}
+
+func TestBuilderSpecWithUnknownMetricOrFieldIs422(t *testing.T) {
+	e, _ := ruleEnv(t)
+	for name, body := range map[string]string{
+		"unknown metric": `{"name":"a","kind":"metric","severity":"info","spec":{"metric":{"metric":"nope","op":">","threshold":1}}}`,
+		"hostile metric": `{"name":"a","kind":"metric","severity":"info","spec":{"metric":{"metric":"x\"} or vector(1)","op":">","threshold":1}}}`,
+		"bad op":         `{"name":"a","kind":"metric","severity":"info","spec":{"metric":{"metric":"memTotalReal","op":"=~","threshold":1}}}`,
+		"unknown field":  `{"name":"a","kind":"inventory","severity":"info","spec":{"inventory":{"conditions":[{"field":"password","op":"empty"}]}}}`,
+		"wrong op type":  `{"name":"a","kind":"inventory","severity":"info","spec":{"inventory":{"conditions":[{"field":"site","op":">","value":"1"}]}}}`,
+		"bad regex":      `{"name":"a","kind":"inventory","severity":"info","spec":{"inventory":{"conditions":[{"field":"site","op":"matches","value":"("}]}}}`,
+		"bad template":   `{"name":"a","kind":"metric","severity":"info","summary":"{{.Nope}}","spec":{"metric":{"metric":"memTotalReal","op":">","threshold":1}}}`,
+		"bad for":        `{"name":"a","kind":"metric","severity":"info","for":"soon","spec":{"metric":{"metric":"memTotalReal","op":">","threshold":1}}}`,
+		"kind mismatch":  `{"name":"a","kind":"inventory","severity":"info","spec":{"metric":{"metric":"memTotalReal","op":">","threshold":1}}}`,
+	} {
+		if rec := e.do("POST", "/api/v1/alert-rules", body); rec.Code != http.StatusUnprocessableEntity {
+			t.Errorf("%s: %d %s", name, rec.Code, rec.Body)
+		}
+	}
+	if rules := listRules(t, e); len(rules) != 1 {
+		t.Fatalf("a refused rule was stored: %+v", rules)
+	}
+}
+
+func listRules(t *testing.T, e *env) []api.AlertRuleInfo {
+	t.Helper()
+	var l api.AlertRuleList
+	e.getJSON(t, "/api/v1/alert-rules", &l)
+	return l.Items
+}
+
+func TestBuiltinRulesCannotBeEditedOrDeleted(t *testing.T) {
+	e, _ := ruleEnv(t)
+	isProblem(t, e.do("PUT", "/api/v1/alert-rules/cpu_high", cpuRule), http.StatusConflict)
+	isProblem(t, e.do("DELETE", "/api/v1/alert-rules/cpu_high", ""), http.StatusConflict)
+	reserved := strings.Replace(cpuRule, "cpu_server", "cpu_high", 1)
+	isProblem(t, e.do("POST", "/api/v1/alert-rules", reserved), http.StatusConflict)
+	var got api.AlertRuleInfo
+	e.getJSON(t, "/api/v1/alert-rules/cpu_high", &got)
+	if got.Editable || got.Kind != "builtin" {
+		t.Fatalf("builtin = %+v", got)
+	}
+}
+
+func TestDeletingOrReshapingAReferencedRuleIs409(t *testing.T) {
+	e, _ := ruleEnv(t)
+	v := createRule(t, e, cpuRule)
+	p := &store.AlertPolicy{Name: "servers", Rule: "cpu_server", Action: "alert", Enabled: true, Scope: `{"all":true}`}
+	if err := e.st.CreateAlertPolicy(t.Context(), p); err != nil {
+		t.Fatal(err)
+	}
+	rec := e.do("DELETE", "/api/v1/alert-rules/"+v.ID, "")
+	isProblem(t, rec, http.StatusConflict)
+	if !strings.Contains(rec.Body.String(), "servers") {
+		t.Fatalf("409 does not name the policy: %s", rec.Body)
+	}
+	renamed := strings.Replace(cpuRule, "cpu_server", "cpu_renamed", 1)
+	isProblem(t, e.do("PUT", "/api/v1/alert-rules/"+v.ID, renamed), http.StatusConflict)
+	// Turning it into an interface rule would invalidate policies written for a device rule.
+	inv := `{"name":"cpu_server","kind":"inventory","severity":"warning","spec":{"inventory":{"conditions":[{"field":"if_alias","op":"empty"}]}}}`
+	isProblem(t, e.do("PUT", "/api/v1/alert-rules/"+v.ID, inv), http.StatusConflict)
+	if err := e.st.DeleteAlertPolicy(t.Context(), p.ID); err != nil {
+		t.Fatal(err)
+	}
+	if rec := e.do("DELETE", "/api/v1/alert-rules/"+v.ID, ""); rec.Code != http.StatusNoContent {
+		t.Fatalf("DELETE after the policy is gone: %d %s", rec.Code, rec.Body)
+	}
+}
+
+func TestCopyBuiltinMakesACustomRuleOrExplains(t *testing.T) {
+	e, _ := ruleEnv(t)
+	rec := e.do("POST", "/api/v1/alert-rules/cpu_high/copy", `{}`)
+	if rec.Code != http.StatusCreated {
+		t.Fatalf("copy: %d %s", rec.Code, rec.Body)
+	}
+	var v api.AlertRuleInfo
+	if err := json.Unmarshal(rec.Body.Bytes(), &v); err != nil || v.Name != "cpu_high_custom" || !v.Editable {
+		t.Fatalf("copy = %s", rec.Body)
+	}
+	isProblem(t, e.do("POST", "/api/v1/alert-rules/cpu_high/copy", `{}`), http.StatusConflict)
+	if rec := e.do("POST", "/api/v1/alert-rules/cpu_high/copy", `{"name":"mine"}`); rec.Code != http.StatusCreated {
+		t.Fatalf("named copy: %d %s", rec.Code, rec.Body)
+	}
+	rec = e.do("POST", "/api/v1/alert-rules/disk_full/copy", `{}`)
+	if rec.Code != http.StatusNotFound { // the fake lists only cpu_high
+		t.Fatalf("unknown: %d %s", rec.Code, rec.Body)
+	}
+	isProblem(t, e.do("POST", "/api/v1/alert-rules/"+v.ID+"/copy", `{}`), http.StatusConflict)
+}
+
+func TestCatalogueAndTestRoutes(t *testing.T) {
+	e, _ := ruleEnv(t)
+	var c api.RuleCatalogue
+	e.getJSON(t, "/api/v1/alert-rules/catalogue", &c)
+	if len(c.Metrics) != 2 || len(c.Fields) == 0 || len(c.Ops) != 6 {
+		t.Fatalf("catalogue = %+v", c)
+	}
+	rec := e.do("POST", "/api/v1/alert-rules/test", cpuRule)
+	var res api.RuleTestResult
+	if err := json.Unmarshal(rec.Body.Bytes(), &res); rec.Code != http.StatusOK || err != nil || res.Breaching != 1 ||
+		res.Expr != `max by (device) (last_over_time(hrProcessorLoad[10m]))` {
+		t.Fatalf("test = %d %s", rec.Code, rec.Body)
+	}
+	isProblem(t, e.do("POST", "/api/v1/alert-rules/test", strings.Replace(cpuRule, "hrProcessorLoad", "nope", 1)), http.StatusUnprocessableEntity)
+	if rules := listRules(t, e); len(rules) != 1 {
+		t.Fatalf("a test stored a rule: %+v", rules)
+	}
+}
+
+func TestAlertRuleRoutesAnswer503WithoutARegistry(t *testing.T) {
+	e := newEnvDeps(t, 0, nil)
+	isProblem(t, e.do("GET", "/api/v1/alert-rules/catalogue", ""), http.StatusServiceUnavailable)
+	isProblem(t, e.do("POST", "/api/v1/alert-rules", cpuRule), http.StatusServiceUnavailable)
+}

internal/api/api.go

@@ -63,6 +63,8 @@ type Deps struct {
 	PollerStatus func() []poller.DeviceStatus
 	// AfterDiscover is called after POST /devices/{slug}/discover succeeded, e.g. to make the poller reload that device.
 	AfterDiscover func(deviceID int64)
+	// PollTools serves poll-now, the poll test and the SNMP debug recording. *poller.Poller implements it; nil answers 503.
+	PollTools PollTools
 
 	// DiscoveryTimeout overrides the 30 s seed discovery limit (tests).
 	DiscoveryTimeout time.Duration
@@ -82,8 +84,11 @@ type Deps struct {
 	// group flag, group membership), so the evaluator sees it at once instead of after its cache expires.
 	OnAlertDownChange func()
 
-	// PollInterval is the normal poll cadence; a device unheard of for three of them is down. Default 60 s.
+	// PollInterval is the default poll cadence (300 s); a device unheard of for three of its own is down.
+	// A device's poll_interval_s overrides it.
 	PollInterval time.Duration
+	// Cadence, when set, is the live default poll interval and overrides PollInterval.
+	Cadence *store.PollCadence
 
 	// HotRetention, FiveMinuteRetention and LongRetention are the stores' retention periods. They decide where a
 	// graph stitches and what GET /meta reports. Defaults: 30 d, 120 d, 2 y.
@@ -112,6 +117,10 @@ type Deps struct {
 	APITokens APITokenAdmin
 	// Settings reads and replaces config.yaml and reloads it.
 	Settings SettingsAdmin
+	// Setup drives the first-install wizard; nil answers 503.
+	Setup SetupAdmin
+	// Notify sends test notifications for POST /notify/test; nil answers 503.
+	Notify NotifyTester
 	// Backups runs an online backup for POST /admin/backup.
 	Backups BackupRunner
 	// Stores restarts a failed metric store.
@@ -122,6 +131,8 @@ type Deps struct {
 	PurgeAfter func() time.Duration
 	// AlertPolicy is the running policy engine; routes under /alert-policies and /alert-rules answer 503 without it.
 	AlertPolicy AlertPolicyHost
+	// AlertRules is the rule registry (built-in, config and custom rules); the custom rule routes answer 503 without it.
+	AlertRules AlertRuleHost
 	// OnImported is told the ids of devices a CSV import created, so they can be discovered at once.
 	OnImported func(ids []int64)
 
@@ -151,7 +162,7 @@ func (d Deps) withDefaults() Deps {
 		d.Logger = slog.Default()
 	}
 	d.DiscoveryTimeout = cmp.Or(d.DiscoveryTimeout, DiscoveryTimeout)
-	d.PollInterval = cmp.Or(d.PollInterval, time.Minute)
+	d.PollInterval = cmp.Or(d.PollInterval, store.DefaultPollInterval)
 	d.HotRetention = cmp.Or(d.HotRetention, 30*day)
 	d.FiveMinuteRetention = cmp.Or(d.FiveMinuteRetention, 120*day)
 	d.LongRetention = cmp.Or(d.LongRetention, 2*365*day)
@@ -163,7 +174,7 @@ func (d Deps) withDefaults() Deps {
 		d.Dependencies = storeGraph{src: d.Store}
 	}
 	if d.Reachability == nil {
-		d.Reachability = &persistedReachability{q: d.Querier, clk: d.Clock, interval: d.PollInterval}
+		d.Reachability = &persistedReachability{q: d.Querier, clk: d.Clock, cadence: d.Cadence, interval: d.PollInterval}
 	}
 	return d
 }
@@ -201,16 +212,19 @@ func (s *server) register(api huma.API) {
 	s.registerInterfaces(api)
 	s.registerAlertDown(api)
 	s.registerGraph(api)
+	s.registerGraphCounters(api)
 	s.registerDeviceMetrics(api)
 	s.registerOverview(api)
 	s.registerPrinters(api)
 	s.registerOps(api)
+	s.registerPollTools(api)
 	s.registerHealth(api)
 	s.registerReviews(api)
 	s.registerQuery(api)
 	s.registerMeta(api)
 	s.registerAlerts(api)
 	s.registerAlertPolicies(api)
+	s.registerAlertRules(api)
 	s.registerMaintenance(api)
 	s.registerMetrics(api)
 	s.registerTokens(api)
@@ -223,6 +237,9 @@ func (s *server) register(api huma.API) {
 	s.registerProxmox(api)
 	s.registerMe(api)
 	s.registerSettings(api)
+	s.registerSettingsForm(api)
+	s.registerNotify(api)
+	s.registerSetup(api)
 	s.registerImport(api)
 	s.registerPurge(api)
 	s.registerL2(api)

internal/api/contract_test.go

@@ -399,7 +399,7 @@ func TestResponseSchemasExposeNoIDsOrCredentials(t *testing.T) {
 				if strings.HasSuffix(path, "schemas/TokenCreated") && name == "secret" {
 					continue
 				}
-				if strings.Contains(lower, bad) {
+				if strings.Contains(lower, bad) && !namesAStoredSecret(path, name) {
 					t.Errorf("%s.%s looks like a credential", path, name)
 				}
 			}
@@ -437,3 +437,18 @@ func TestCheckedInOpenAPIIsCurrent(t *testing.T) {
 		t.Fatal("openapi.json is stale: run `go run ./cmd/nmon openapi > openapi.json`")
 	}
 }
+
+// namesAStoredSecret exempts the settings-form fields that hold the NAME of a stored secret (or a Proxmox token
+// identifier), never its value: the value is write-only through PUT /secrets/{name}.
+func namesAStoredSecret(path, name string) bool {
+	for schema, fields := range map[string][]string{
+		"schemas/OIDCForm":           {"client_secret_name"},
+		"schemas/ProxmoxClusterForm": {"secret_name", "token_id"},
+		"schemas/SetupDefaults":      {"oidc_secret_name"},
+	} {
+		if strings.HasSuffix(path, schema) && slices.Contains(fields, name) {
+			return true
+		}
+	}
+	return false
+}

internal/api/detail_test.go

@@ -48,7 +48,7 @@ func TestDeviceDetailLiveRates(t *testing.T) {
 			rateQueries = append(rateQueries, c.expr)
 		}
 	}
-	if len(rateQueries) != 1 || !strings.Contains(rateQueries[0], "rate(ifHCInOctets{device=") || !strings.Contains(rateQueries[0], "[5m])*8") {
+	if len(rateQueries) != 1 || !strings.Contains(rateQueries[0], "rate(ifHCInOctets{device=") || !strings.Contains(rateQueries[0], "[600s])*8") {
 		t.Errorf("want one batched rate query, got %q", rateQueries)
 	}
 	if d.Slug != core.Slug || d.HWSerial != "SN1" || d.SysDescr != "JunOS" || d.UptimeSeconds == nil || *d.UptimeSeconds != 86400 ||

internal/api/device_metrics_graph.go

@@ -45,12 +45,12 @@ func valueExpr(m profile.Metric, slug, comp string) string {
 
 // graphExprs returns the series expression and, for a metric with a capacity that is not drawn as
 // percent, the capacity expression.
-func graphExprs(res *profile.Resolved, m profile.Metric, slug, comp string, step time.Duration) (expr, maxExpr string) {
+func graphExprs(res *profile.Resolved, m profile.Metric, slug, comp string, step, every time.Duration) (expr, maxExpr string) {
 	display := m.EffectiveDisplay()
 	val := valueExpr(m, slug, comp)
 	counter := m.Type == profile.Counter32 || m.Type == profile.Counter64
 	if counter && display == "rate" {
-		val = `rate(` + metricSel(m.Name, slug, comp) + `[` + secs(max(step, minRateWindow)) + `])`
+		val = `rate(` + metricSel(m.Name, slug, comp) + `[` + secs(max(step, rateWindowFor(every))) + `])`
 	}
 	om, hasOf := res.Metric(m.Of)
 	if !hasOf {
@@ -85,7 +85,7 @@ func (s *server) metricGraph(ctx context.Context, in *metricGraphIn) (*struct{ B
 	if in.Aggregate != "" && comp == "" {
 		comp = allComponents
 	}
-	expr, maxExpr := graphExprs(res, m, d.Slug, comp, step)
+	expr, maxExpr := graphExprs(res, m, d.Slug, comp, step, d.PollInterval(s.pollEvery()))
 	if comp == allComponents {
 		expr, maxExpr = in.Aggregate+`(`+expr+`)`, ""
 	}
@@ -122,27 +122,32 @@ func unitOfMetric(res *profile.Resolved, m profile.Metric) string {
 
 // metricPlan applies the /graph range defaults and step rules, limited to the hot store.
 func (s *server) metricPlan(in *metricGraphIn) (*graphPlan, error) {
-	to := in.To
+	p, _, err := s.hotPlan(in.From, in.To, in.Step)
+	return p, err
+}
+
+// hotPlan applies the /graph range defaults (to = now, from = an hour earlier) and step rules to a range that
+// only the hot store answers: a from before hot retention is moved up to its start, and clamped says so.
+func (s *server) hotPlan(from, to time.Time, step int64) (p *graphPlan, clamped bool, err error) {
 	if to.IsZero() {
 		to = s.Clock.Now()
 	}
-	from := in.From
 	if from.IsZero() {
 		from = to.Add(-time.Hour)
 	}
 	hotStart := s.Clock.Now().Add(-s.HotRetention)
 	from = from.UTC()
 	if from.Before(hotStart) {
-		from = hotStart
+		from, clamped = hotStart, true
 	}
 	if !from.Before(to) {
-		return nil, huma.Error422UnprocessableEntity("from must be before to, within the hot retention")
+		return nil, clamped, huma.Error422UnprocessableEntity("from must be before to, within the hot retention")
 	}
-	plan, err := planGraph(from, to.UTC(), hotStart, time.Duration(in.Step)*time.Second)
+	p, err = planGraph(from, to.UTC(), hotStart, time.Duration(step)*time.Second)
 	if err != nil {
-		return nil, huma.Error422UnprocessableEntity(err.Error())
+		return nil, clamped, huma.Error422UnprocessableEntity(err.Error())
 	}
-	return plan, nil
+	return p, clamped, nil
 }
 
 // fetchMetricSeries runs the expression over the plan's hot windows and returns ts -> value.

internal/api/device_poll_interval_test.go

@@ -0,0 +1,30 @@
+package api_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/api"
+)
+
+func TestPutDeviceSetsAndClearsPollInterval(t *testing.T) {
+	e := newEnv(t, 0)
+	d := e.device(t, "A", "yyz", "core")
+	url := "/api/v1/devices/" + d.Slug
+	var v api.DeviceView
+	rec := e.do("PUT", url, `{"poll_interval_s":60}`)
+	_ = json.Unmarshal(rec.Body.Bytes(), &v)
+	if rec.Code != 200 || v.PollIntervalS != 60 || v.PollIntervalEffectiveS != 60 {
+		t.Fatalf("set: %d %+v", rec.Code, v)
+	}
+	if e.ref.n.Load() != 1 {
+		t.Error("poller not refreshed")
+	}
+	rec = e.do("PUT", url, `{"poll_interval_s":0}`)
+	_ = json.Unmarshal(rec.Body.Bytes(), &v)
+	if rec.Code != 200 || v.PollIntervalS != 0 || v.PollIntervalEffectiveS != 300 {
+		t.Fatalf("clear: %d %+v", rec.Code, v)
+	}
+	isProblem(t, e.do("PUT", url, `{"poll_interval_s":10}`), 422)
+	isProblem(t, e.do("PUT", url, `{"poll_interval_s":7200}`), 422)
+}

internal/api/device_put.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"errors"
+	"fmt"
 	"strings"
 
 	"github.com/danielgtaylor/huma/v2"
@@ -13,13 +14,14 @@ import (
 // PutDeviceBody is the body of PUT /api/v1/devices/{slug}. Absent fields stay
 // as they are; an empty parents list removes every parent.
 type PutDeviceBody struct {
-	Name    *string   `json:"name,omitempty" minLength:"1" maxLength:"255"`
-	Site    *string   `json:"site,omitempty"`
-	Role    *string   `json:"role,omitempty"`
-	Enabled *bool     `json:"enabled,omitempty"`
-	Parents []string  `json:"parents,omitempty" doc:"Parent device slugs, replacing the current parents."`
-	SNMP    *SNMPBody `json:"snmp,omitempty" doc:"Replaces the SNMP credential; secrets are write-only."`
-	Profile *string   `json:"profile,omitempty" doc:"A profile name pins it; an empty string returns the device to automatic matching. A change triggers one discovery."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
+	Name          *string   `json:"name,omitempty" minLength:"1" maxLength:"255"`
+	Site          *string   `json:"site,omitempty"`
+	Role          *string   `json:"role,omitempty"`
+	Enabled       *bool     `json:"enabled,omitempty"`
+	Parents       []string  `json:"parents,omitempty" doc:"Parent device slugs, replacing the current parents."`
+	SNMP          *SNMPBody `json:"snmp,omitempty" doc:"Replaces the SNMP credential; secrets are write-only."`
+	PollIntervalS *int      `json:"poll_interval_s,omitempty" minimum:"0" maximum:"3600" doc:"Poll interval, 30 to 3600 s; 0 = default."`
+	Profile       *string   `json:"profile,omitempty" doc:"A profile name pins it; an empty string returns the device to automatic matching. A change triggers one discovery."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
 }
 
 func isNotFound(err error) bool { return errors.Is(err, store.ErrNotFound) }
@@ -90,6 +92,9 @@ func (s *server) planEdit(ctx context.Context, dev *store.Device, b PutDeviceBod
 		p.setParents, p.parents = true, parents
 		p.change["parents"] = b.Parents
 	}
+	if err := planPollInterval(p, b.PollIntervalS); err != nil {
+		return nil, err
+	}
 	if b.Profile != nil {
 		if err := s.planProfile(p, dev, *b.Profile); err != nil {
 			return nil, err
@@ -110,6 +115,20 @@ func (s *server) planEdit(ctx context.Context, dev *store.Device, b PutDeviceBod
 	return p, nil
 }
 
+// planPollInterval validates the poll-interval override: 0 clears it, otherwise it must be in bounds.
+func planPollInterval(p *editPlan, v *int) error {
+	if v == nil || *v == p.after.PollIntervalS {
+		return nil
+	}
+	if *v != 0 && (*v < store.MinPollIntervalS || *v > store.MaxPollIntervalS) {
+		return huma.Error422UnprocessableEntity(fmt.Sprintf("poll_interval_s must be 0 or %d to %d",
+			store.MinPollIntervalS, store.MaxPollIntervalS))
+	}
+	p.after.PollIntervalS = *v
+	p.change["poll_interval_s"] = *v
+	return nil
+}
+
 // applyEdit writes the plan and its audit row in one transaction. The old
 // credential is deleted once nothing uses it.
 func (s *server) applyEdit(ctx context.Context, dev *store.Device, p *editPlan) error {

internal/api/device_view.go

@@ -45,9 +45,9 @@ type InterfaceView struct {
 	LastDiscovered time.Time  `json:"last_discovered"`
 	Unreliable     bool       `json:"unreliable" doc:"Counters cannot be trusted (32-bit counters on a fast link); no rate is reported."`
 	RetiredAt      *time.Time `json:"retired_at" nullable:"true"`
-	InBPS          *float64   `json:"in_bps" nullable:"true" doc:"Inbound bits per second over 5 minutes."`
+	InBPS          *float64   `json:"in_bps" nullable:"true" doc:"Inbound bits per second over the live window (two default poll intervals)."`
 	OutBPS         *float64   `json:"out_bps" nullable:"true"`
-	InErrorsPS     *float64   `json:"in_errors_ps" nullable:"true" doc:"Inbound errors per second over 5 minutes."`
+	InErrorsPS     *float64   `json:"in_errors_ps" nullable:"true" doc:"Inbound errors per second over the live window."`
 	OutErrorsPS    *float64   `json:"out_errors_ps" nullable:"true"`
 	AlertDownState
 }
@@ -63,23 +63,27 @@ type DeviceView struct {
 	ProfilePinned bool              `json:"profile_pinned" doc:"An operator chose the profile; discovery keeps it."`
 	ProfileMatch  *ProfileMatchView `json:"profile_match,omitempty"`
 	Enabled       bool              `json:"enabled"`
-	MgmtIP        string            `json:"mgmt_ip"`
-	Addresses     []string          `json:"addresses"`
-	SNMPVersion   string            `json:"snmp_version,omitempty"`
-	SysName       string            `json:"sys_name"`
-	SysObjectID   string            `json:"sys_object_id"`
-	SysDescr      string            `json:"sys_descr"`
-	HWSerial      string            `json:"hw_serial"`
-	UptimeSeconds *float64          `json:"uptime_seconds" nullable:"true" doc:"sysUpTime from the hot store; null when unknown."`
-	Reachability  string            `json:"reachability" enum:"up,down,warning,unknown,maintenance"`
-	PollState     string            `json:"poll_state" enum:"ok,degraded,backoff"`
-	Warnings      []string          `json:"warnings"`
-	LastSeen      *time.Time        `json:"last_seen" nullable:"true"`
-	CreatedAt     time.Time         `json:"created_at"`
-	RetiredAt     *time.Time        `json:"retired_at,omitempty" nullable:"true"`
-	Parents       []DeviceRef       `json:"parents"`
-	Groups        []GroupView       `json:"groups"`
-	Interfaces    []InterfaceView   `json:"interfaces"`
+	// PollIntervalS is the admin's SNMP poll interval override in seconds (0 = the default);
+	// PollIntervalEffectiveS is the interval in force.
+	PollIntervalS          int             `json:"poll_interval_s" doc:"Poll interval override in seconds; 0 means the default."`
+	PollIntervalEffectiveS int             `json:"poll_interval_effective_s" doc:"The SNMP poll interval in force, in seconds."`
+	MgmtIP                 string          `json:"mgmt_ip"`
+	Addresses              []string        `json:"addresses"`
+	SNMPVersion            string          `json:"snmp_version,omitempty"`
+	SysName                string          `json:"sys_name"`
+	SysObjectID            string          `json:"sys_object_id"`
+	SysDescr               string          `json:"sys_descr"`
+	HWSerial               string          `json:"hw_serial"`
+	UptimeSeconds          *float64        `json:"uptime_seconds" nullable:"true" doc:"sysUpTime from the hot store; null when unknown."`
+	Reachability           string          `json:"reachability" enum:"up,down,warning,unknown,maintenance"`
+	PollState              string          `json:"poll_state" enum:"ok,degraded,backoff"`
+	Warnings               []string        `json:"warnings"`
+	LastSeen               *time.Time      `json:"last_seen" nullable:"true"`
+	CreatedAt              time.Time       `json:"created_at"`
+	RetiredAt              *time.Time      `json:"retired_at,omitempty" nullable:"true"`
+	Parents                []DeviceRef     `json:"parents"`
+	Groups                 []GroupView     `json:"groups"`
+	Interfaces             []InterfaceView `json:"interfaces"`
 	// M5 additions.
 	HostedOn     *DeviceRef   `json:"hosted_on,omitempty" doc:"The host node of a VM or container."`
 	Guests       []DeviceRef  `json:"guests" doc:"VMs and containers hosted on this device."`
@@ -123,6 +127,7 @@ func (s *server) view(ctx context.Context, slug string) (*deviceOut, error) {
 	v := DeviceView{
 		Slug: d.Slug, Name: d.Name, Kind: d.Kind, Site: d.Site, Role: d.Role, Profile: d.Profile,
 		ProfilePinned: d.ProfilePinned, ProfileMatch: s.profileMatch(d), Enabled: d.Enabled,
+		PollIntervalS: d.PollIntervalS, PollIntervalEffectiveS: int(d.PollInterval(s.pollEvery()) / time.Second),
 		MgmtIP: d.MgmtIP, SysName: d.SysName, SysObjectID: d.SysObjectID, SysDescr: d.SysDescr, HWSerial: d.HWSerial,
 		PollState: pollStateOf(d), Warnings: s.warningsOf(d), LastSeen: utcp(d.LastSeen), CreatedAt: d.CreatedAt.UTC(),
 		RetiredAt: utcp(d.RetiredAt), Addresses: []string{}, Parents: []DeviceRef{}, Groups: []GroupView{},

internal/api/graph.go

@@ -19,7 +19,8 @@ func (s *server) registerGraph(api huma.API) {
 			"hourly rollups, and a range that needs both is stitched at stitched_at. Stretches with no data in either store " +
 			"are listed in partial. An unreliable interface returns no series. max=true adds the per-step maximum; " +
 			"p95=true adds the 95th percentile of 5-minute averages: the monthly figure for a whole calendar month, otherwise " +
-			"computed live over the range; source says which.",
+			"computed live over the range; source says which. totals integrates the average series into bytes in and out " +
+			"over the range; covered_seconds says how much of it had data.",
 		Errors: []int{http.StatusNotFound, http.StatusUnprocessableEntity, http.StatusTooManyRequests, http.StatusServiceUnavailable},
 	}, s.graph)
 }
@@ -30,7 +31,7 @@ type graphIn struct {
 	To    time.Time `query:"to" doc:"End, RFC 3339. Default now."`
 	Step  int64     `query:"step" minimum:"1" doc:"Step in seconds. Default max(60, range/1000) rounded up to whole minutes."`
 	Dir   string    `query:"dir" enum:"in,out" doc:"Only this direction; the other series is empty."`
-	Max   bool      `query:"max" doc:"Also return in_max and out_max, the highest one-minute rate within each step."`
+	Max   bool      `query:"max" doc:"Also return in_max and out_max, the highest poll-to-poll rate within each step."`
 	P95   bool      `query:"p95" doc:"Also return the 95th percentile: monthly for a whole calendar month, else live over the range."`
 	Month string    `query:"month" pattern:"^\\d{4}-(0[1-9]|1[0-2])$" doc:"YYYY-MM in the site timezone. Replaces from and to with that month's bounds."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
 }
@@ -64,7 +65,8 @@ func (s *server) graph(ctx context.Context, in *graphIn) (*struct{ Body GraphRes
 		resp.Unreliable, resp.UnreliableReason = true, unreliableReason(i)
 		return &struct{ Body GraphResponse }{resp}, nil
 	}
-	if err := s.fillGraph(ctx, &resp, i, plan, in, graphSite{hotStart: hotStart, site: dev.Site}); err != nil {
+	if err := s.fillGraph(ctx, &resp, i, plan, in, graphSite{hotStart: hotStart, site: dev.Site,
+		every: dev.PollInterval(s.pollEvery()), from: from, to: to}); err != nil {
 		return nil, err
 	}
 	return &struct{ Body GraphResponse }{resp}, nil
@@ -97,6 +99,8 @@ func (s *server) graphRange(in *graphIn, site string) (from, to time.Time, err e
 type graphSite struct {
 	hotStart time.Time
 	site     string
+	every    time.Duration // the device's effective poll interval
+	from, to time.Time     // the requested range, unaligned
 }
 
 // fillGraph queries the stores the plan names and lays the samples out on the grid.
@@ -110,7 +114,7 @@ func (s *server) fillGraph(
 	defer release()
 	qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
 	defer cancel()
-	q := &graphQuery{iface: i, dirs: []string{dirIn, dirOut}, withMax: in.Max}
+	q := &graphQuery{iface: i, dirs: []string{dirIn, dirOut}, withMax: in.Max, every: gs.every}
 	if in.Dir != "" {
 		q.dirs = []string{in.Dir}
 	}
@@ -121,6 +125,7 @@ func (s *server) fillGraph(
 		}
 	}
 	assemble(resp, p, q, data)
+	resp.Totals = graphTotals(p, q.dirs, data.avg, gs.from, gs.to)
 	if in.P95 {
 		p95, err := s.graphP95(qctx, q, p, gs)
 		if err != nil {

internal/api/graph_counters.go

@@ -0,0 +1,91 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+)
+
+// counterKinds maps a graph kind to the polled counters that carry its in and out directions and the unit of
+// the rate. Only counters every profile polls for every port are listed (profiles/generic.yaml); a kind joins
+// the list when its counters do. Packet and broadcast/multicast counters are not polled today.
+var counterKinds = map[string]struct{ in, out, unit string }{
+	"errors":   {"ifInErrors", "ifOutErrors", "errors/s"},
+	"discards": {"ifInDiscards", "ifOutDiscards", "discards/s"},
+}
+
+func (s *server) registerGraphCounters(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "get-graph-counters", Method: http.MethodGet, Path: "/api/v1/graph/counters", Tags: []string{"graph"},
+		Summary: "Errors or discards per second in and out of one interface",
+		Description: "The rate of the interface's error or discard counters, from the hot store only: these counters have no " +
+			"hourly rollup, so a range that starts before hot retention is moved up to its start and clamped says so. " +
+			"The same step and rate-window rules as /graph apply.",
+		Errors: []int{http.StatusNotFound, http.StatusUnprocessableEntity, http.StatusTooManyRequests, http.StatusServiceUnavailable},
+	}, s.graphCounters)
+}
+
+type graphCountersIn struct {
+	Iface string    `query:"iface" required:"true" minLength:"1" doc:"Interface ULID."`
+	Kind  string    `query:"kind" required:"true" enum:"errors,discards" doc:"Which counters."`
+	From  time.Time `query:"from" doc:"Start, RFC 3339. Default one hour before to."`
+	To    time.Time `query:"to" doc:"End, RFC 3339. Default now."`
+	Step  int64     `query:"step" minimum:"1" doc:"Step in seconds. Default max(60, range/1000) rounded up to whole minutes."`
+}
+
+// CounterSeries holds the per-second rates of one counter pair.
+type CounterSeries struct {
+	In  []Point `json:"in"`
+	Out []Point `json:"out"`
+}
+
+// CounterGraphResponse is a rate graph of an interface's error or discard counters.
+type CounterGraphResponse struct {
+	Iface       GraphIface    `json:"iface"`
+	Kind        string        `json:"kind" enum:"errors,discards"`
+	Unit        string        `json:"unit" doc:"Unit of the series values, for example errors/s."`
+	StepSeconds int64         `json:"step_seconds"`
+	From        time.Time     `json:"from" doc:"Start of the answered range, after any clamp to hot retention."`
+	To          time.Time     `json:"to"`
+	Clamped     bool          `json:"clamped" doc:"The requested start was before hot retention and was moved up to it."`
+	Series      CounterSeries `json:"series" doc:"[unix_s, per-second rate|null] pairs."`
+}
+
+func (s *server) graphCounters(ctx context.Context, in *graphCountersIn) (*struct{ Body CounterGraphResponse }, error) {
+	i, err := s.Store.GetInterface(ctx, in.Iface)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	dev, err := s.Store.GetDeviceByID(ctx, i.DeviceID)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	plan, clamped, err := s.hotPlan(in.From, in.To, in.Step)
+	if err != nil {
+		return nil, err
+	}
+	kind := counterKinds[in.Kind]
+	step := plan.newest().step
+	every := dev.PollInterval(s.pollEvery())
+	grid := gridOf(plan)
+	resp := CounterGraphResponse{
+		Iface: GraphIface{
+			ID: i.ID, IfName: i.IfName, IfAlias: i.IfAlias, SpeedBPS: speedBPS(i), Device: DeviceRef{Slug: dev.Slug, Name: dev.Name},
+		},
+		Kind: in.Kind, Unit: kind.unit, StepSeconds: int64(step / time.Second), From: plan.from, To: plan.to, Clamped: clamped,
+	}
+	for _, c := range []struct {
+		name string
+		dst  *[]Point
+	}{{kind.in, &resp.Series.In}, {kind.out, &resp.Series.Out}} {
+		expr := `rate(` + c.name + `{iface=` + quote(i.ID) + `}[` + secs(max(step, rateWindowFor(every))) + `])`
+		series, err := s.fetchMetricSeries(ctx, expr, plan)
+		if err != nil {
+			return nil, err
+		}
+		*c.dst = pointsOf(grid, series)
+	}
+	return &struct{ Body CounterGraphResponse }{resp}, nil
+}

internal/api/graph_counters_test.go

@@ -0,0 +1,49 @@
+package api_test
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/api"
+)
+
+func countersURL(id, kind string, from, to time.Time) string {
+	return "/api/v1/graph/counters?iface=" + id + "&kind=" + kind + "&from=" + url.QueryEscape(iso(from)) + "&to=" + url.QueryEscape(iso(to))
+}
+
+func TestGraphCountersQueryTheErrorCountersAsRates(t *testing.T) {
+	e, i := newGraphEnv(t, &graphVM{hotVal: rising})
+	var g api.CounterGraphResponse
+	rec := e.getJSON(t, countersURL(i.ID, "errors", t0.Add(-time.Hour), t0), &g)
+	assertNoLeaks(t, "counters", rec.Body.Bytes())
+	if g.Kind != "errors" || g.Unit != "errors/s" || g.StepSeconds != 60 || g.Clamped || g.Iface.ID != i.ID {
+		t.Fatalf("response: %+v", g)
+	}
+	if len(g.Series.In) != 61 || len(g.Series.Out) != 61 || g.Series.In[60].TS != t0.Unix() {
+		t.Fatalf("series: %d in, %d out", len(g.Series.In), len(g.Series.Out))
+	}
+	qs := e.q.queries()
+	if len(qs) != 2 || qs[0].store != "hot" || qs[0].expr != `rate(ifInErrors{iface="`+i.ID+`"}[600s])` ||
+		qs[1].expr != `rate(ifOutErrors{iface="`+i.ID+`"}[600s])` {
+		t.Errorf("queries: %+v", qs)
+	}
+}
+
+func TestGraphCountersDiscardsAndHotOnlyClamp(t *testing.T) {
+	e, i := newGraphEnv(t, &graphVM{hotVal: rising})
+	var g api.CounterGraphResponse
+	e.getJSON(t, countersURL(i.ID, "discards", t0.Add(-90*24*time.Hour), t0), &g)
+	if g.Kind != "discards" || !g.Clamped || !g.From.After(t0.Add(-90*24*time.Hour)) {
+		t.Fatalf("a range older than hot retention must be clamped: %+v", g)
+	}
+	for _, q := range e.q.queries() {
+		if q.store != "hot" || !strings.Contains(q.expr, "Discards") {
+			t.Errorf("query: %+v", q)
+		}
+	}
+	isProblem(t, e.do("GET", countersURL(i.ID, "packets", t0.Add(-time.Hour), t0), ""), 422)
+	isProblem(t, e.do("GET", countersURL(i.ID, "errors", t0, t0.Add(-time.Hour)), ""), 422)
+	isProblem(t, e.do("GET", countersURL("01ARZ3NDEKTSV4RRFFQ69G5FAV", "errors", t0.Add(-time.Hour), t0), ""), 404)
+}

internal/api/graph_fetch.go

@@ -15,13 +15,21 @@ import (
 const (
 	dirIn  = "in"
 	dirOut = "out"
-	// minRateWindow keeps the rate window above two poll intervals so a step of
-	// one minute still sees two samples.
+	// minRateWindow is the floor of every rate window, for devices polled faster than once a minute.
 	minRateWindow = 2 * time.Minute
-	// maxInnerStep is the resolution of the max-over-step subquery.
+	// maxInnerStep is the finest resolution of the max-over-step subquery.
 	maxInnerStep = time.Minute
 )
 
+// rateWindowFor is the window of a rate() over a device polled every `every`: two polls, so a window always holds
+// two samples (VictoriaMetrics also takes the one before the window, which only helps). A zero every means the default.
+func rateWindowFor(every time.Duration) time.Duration {
+	if every <= 0 {
+		every = store.DefaultPollInterval
+	}
+	return max(minRateWindow, 2*every)
+}
+
 // graphData holds the samples found, by direction and Unix second.
 type graphData struct {
 	avg map[string]map[int64]float64
@@ -50,6 +58,7 @@ type graphQuery struct {
 	iface   *store.Interface
 	dirs    []string
 	withMax bool
+	every   time.Duration // the device's effective poll interval; sizes the rate windows
 }
 
 // octets is the counter that carries direction dir for the interface.
@@ -70,13 +79,14 @@ func secs(d time.Duration) string { return strconv.FormatInt(int64(d/time.Second
 // rawExpr is the average bits per second of dir over the rate window.
 func (q *graphQuery) rawExpr(dir string, step time.Duration) string {
 	sel := q.octets(dir) + `{iface=` + quote(q.iface.ID) + `}`
-	return `rate(` + sel + `[` + secs(max(step, minRateWindow)) + `])*8`
+	return `rate(` + sel + `[` + secs(max(step, rateWindowFor(q.every))) + `])*8`
 }
 
-// rawMaxExpr is the highest one-minute rate seen within each step.
+// rawMaxExpr is the highest poll-to-poll rate seen within each step. irate is the rate between the last two
+// samples in its window, which is the poll-to-poll rate whatever the cadence.
 func (q *graphQuery) rawMaxExpr(dir string, step time.Duration) string {
 	sel := q.octets(dir) + `{iface=` + quote(q.iface.ID) + `}`
-	return `max_over_time((rate(` + sel + `[` + secs(maxInnerStep) + `])*8)[` + secs(step) + `:` + secs(maxInnerStep) + `])`
+	return `max_over_time((irate(` + sel + `[` + secs(rateWindowFor(q.every)) + `])*8)[` + secs(step) + `:` + secs(maxInnerStep) + `])`
 }
 
 // longExpr reads an hourly rollup metric. A re-run of a window writes the next

internal/api/graph_p95.go

@@ -23,7 +23,9 @@ func (s *server) graphP95(ctx context.Context, q *graphQuery, p *graphPlan, gs g
 		r, err := s.P95.P95(ctx, q.iface.ID, m.Label)
 		switch {
 		case err == nil:
-			return p95Graph(r), nil
+			g := p95Graph(r)
+			g.AggregateBPS = s.liveAggregate(ctx, q, p, gs.hotStart)
+			return g, nil
 		case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
 			return nil, err
 		case !errors.Is(err, p95.ErrNoData):
@@ -46,42 +48,91 @@ func (s *server) wholeMonth(site string, p *graphPlan) (p95.Month, bool) {
 // monthly value; until then the source is "live" and coverage says how much of
 // the range's 5-minute slots had data, so a short history never passes for a full month.
 func (s *server) livep95(ctx context.Context, q *graphQuery, p *graphPlan, hotStart time.Time) (*GraphP95, error) {
-	raw := p.newest()
-	if raw.tier != tierRaw {
-		return &GraphP95{Source: string(p95.SourceLive), Partial: true}, nil
-	}
-	start := ceilTo(latest(raw.from, hotStart, raw.bound.Add(-maxP95Span)), p95Slot)
-	end := floorTo(raw.bound, p95Slot)
-	expected := max(int64(p.to.Sub(p.from)/p95Slot), 1)
 	out := &GraphP95{Source: string(p95.SourceLive)}
-	if end.Before(start) {
+	slots, ok, err := s.p95Slots(ctx, q, p, hotStart)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		out.Partial = p.newest().tier != tierRaw
 		return out, nil
 	}
-	w := tierWindow{tier: tierRaw, from: start, last: end, bound: end, step: p95Slot}
+	expected := max(int64(p.to.Sub(p.from)/p95Slot), 1)
 	coverage := 1.0
 	for _, dir := range q.dirs {
-		var vals []float64
-		expr := `rate(` + q.octets(dir) + `{iface=` + quote(q.iface.ID) + `}[` + secs(p95Slot) + `])*8`
-		err := s.fetchOne(ctx, "hot", expr, w, func(_ int64, v float64) { vals = append(vals, v) })
-		if err != nil {
-			return nil, err
-		}
-		coverage = min(coverage, float64(len(vals))/float64(expected))
-		if len(vals) == 0 {
+		coverage = min(coverage, float64(len(slots[dir]))/float64(expected))
+		if len(slots[dir]) == 0 {
 			continue
 		}
-		v := nearestRank(vals, p95Quantile)
+		v := nearestRank(mapValues(slots[dir]), p95Quantile)
 		if dir == dirIn {
 			out.InBPS = &v
 		} else {
 			out.OutBPS = &v
 		}
 	}
+	out.AggregateBPS = aggregateP95(slots)
 	out.Coverage = min(coverage, 1)
 	out.Partial = out.Coverage < p95.LowCoverage
 	return out, nil
 }
 
+// p95Slots fetches the 5-minute average rates of every requested direction over the raw part of the range, by
+// direction and slot time. ok is false when the range has no raw slots to compute from.
+func (s *server) p95Slots(
+	ctx context.Context, q *graphQuery, p *graphPlan, hotStart time.Time,
+) (slots map[string]map[int64]float64, ok bool, err error) {
+	raw := p.newest()
+	if raw.tier != tierRaw {
+		return nil, false, nil
+	}
+	start := ceilTo(latest(raw.from, hotStart, raw.bound.Add(-maxP95Span)), p95Slot)
+	end := floorTo(raw.bound, p95Slot)
+	if end.Before(start) {
+		return nil, false, nil
+	}
+	slots = map[string]map[int64]float64{}
+	w := tierWindow{tier: tierRaw, from: start, last: end, bound: end, step: p95Slot}
+	for _, dir := range q.dirs {
+		m := map[int64]float64{}
+		expr := `rate(` + q.octets(dir) + `{iface=` + quote(q.iface.ID) + `}[` + secs(max(p95Slot, rateWindowFor(q.every))) + `])*8`
+		if err := s.fetchOne(ctx, "hot", expr, w, func(ts int64, v float64) { m[ts] = v }); err != nil {
+			return nil, false, err
+		}
+		slots[dir] = m
+	}
+	return slots, true, nil
+}
+
+// aggregateP95 is the 95th percentile of in+out per 5-minute slot, the figure LibreNMS calls the aggregate. A slot
+// counts only when both directions have a sample, so a one-sided gap never lowers the sum. Nil when either
+// direction was not requested or no slot has both.
+func aggregateP95(slots map[string]map[int64]float64) *float64 {
+	in, out := slots[dirIn], slots[dirOut]
+	if in == nil || out == nil {
+		return nil
+	}
+	sums := make([]float64, 0, len(in))
+	for ts, a := range in {
+		if b, ok := out[ts]; ok {
+			sums = append(sums, a+b)
+		}
+	}
+	if len(sums) == 0 {
+		return nil
+	}
+	v := nearestRank(sums, p95Quantile)
+	return &v
+}
+
+func mapValues(m map[int64]float64) []float64 {
+	out := make([]float64, 0, len(m))
+	for _, v := range m {
+		out = append(out, v)
+	}
+	return out
+}
+
 // latest returns the latest of ts.
 func latest(ts ...time.Time) time.Time {
 	out := ts[0]
@@ -92,3 +143,14 @@ func latest(ts ...time.Time) time.Time {
 	}
 	return out
 }
+
+// liveAggregate is the aggregate 95th for a figure whose in and out come from the materialised month: the
+// month's own job stores no aggregate, so it is computed from the hot store when the month lies in it (nil
+// otherwise, and when the store fails, since the figure is a bonus next to a good monthly answer).
+func (s *server) liveAggregate(ctx context.Context, q *graphQuery, p *graphPlan, hotStart time.Time) *float64 {
+	slots, ok, err := s.p95Slots(ctx, q, p, hotStart)
+	if err != nil || !ok {
+		return nil
+	}
+	return aggregateP95(slots)
+}

internal/api/graph_test.go

@@ -85,6 +85,10 @@ func TestGraphRawTierWithinHotRetention(t *testing.T) {
 	if g.Partial == nil || len(g.Partial) != 0 {
 		t.Errorf("partial must be [] when complete: %v", g.Partial)
 	}
+	if g.Totals == nil || g.Totals.InBytes == nil || g.Totals.OutBytes == nil || *g.Totals.InBytes <= 0 ||
+		g.Totals.CoveredSeconds != 3600 || g.Totals.RangeSeconds != 3600 {
+		t.Errorf("totals: %+v", g.Totals)
+	}
 	if g.Iface.ID != i.ID || g.Iface.IfName != "xe-0/0/1" || g.Iface.IfAlias != "uplink" || g.Iface.SpeedBPS != 1_000_000_000 || g.Iface.Device.Name != "Core" {
 		t.Errorf("iface: %+v", g.Iface)
 	}
@@ -100,7 +104,7 @@ func TestGraphRawTierWithinHotRetention(t *testing.T) {
 	}
 	qs := e.q.queries()
 	if len(qs) != 2 || qs[0].store != "hot" || qs[0].step != time.Minute ||
-		qs[0].expr != `rate(ifHCInOctets{iface="`+i.ID+`"}[120s])*8` || !strings.Contains(qs[1].expr, "ifHCOutOctets") {
+		qs[0].expr != `rate(ifHCInOctets{iface="`+i.ID+`"}[600s])*8` || !strings.Contains(qs[1].expr, "ifHCOutOctets") {
 		t.Errorf("queries: %+v", qs)
 	}
 	var raw map[string]json.RawMessage
@@ -328,7 +332,7 @@ func TestGraphUnreliableInterfaceHasNoSeries(t *testing.T) {
 			var g api.GraphResponse
 			rec := e.getJSON(t, graphURL(i.ID, t0.Add(-time.Hour), t0, "&max=true&p95=true"), &g)
 			if !g.Unreliable || g.UnreliableReason != tc.reason || len(g.Series.In) != 0 || len(g.Series.Out) != 0 ||
-				g.P95 != nil || g.Partial == nil || g.Tier != "raw" || g.StepSeconds != 60 {
+				g.P95 != nil || g.Totals != nil || g.Partial == nil || g.Tier != "raw" || g.StepSeconds != 60 {
 				t.Errorf("unreliable graph: %+v", g)
 			}
 			if n := len(e.q.queries()); n != 0 {
@@ -348,6 +352,9 @@ func TestGraphDirectionFilterAndMax(t *testing.T) {
 	if len(g.Series.In) != 0 || len(g.Series.Out) != 61 || len(g.Series.OutMax) != 61 || g.Series.InMax != nil {
 		t.Errorf("dir=out: in=%d out=%d out_max=%d", len(g.Series.In), len(g.Series.Out), len(g.Series.OutMax))
 	}
+	if g.Totals == nil || g.Totals.InBytes != nil || g.Totals.OutBytes == nil {
+		t.Errorf("dir=out totals: %+v", g.Totals)
+	}
 	qs := e.q.queries()
 	if len(qs) != 2 {
 		t.Fatalf("queries %d", len(qs))
@@ -357,7 +364,7 @@ func TestGraphDirectionFilterAndMax(t *testing.T) {
 			t.Errorf("dir=out queried the in counter: %s", c.expr)
 		}
 	}
-	if !strings.Contains(qs[1].expr, "max_over_time((rate(ifHCOutOctets{iface=") || !strings.Contains(qs[1].expr, "[60s:60s])") {
+	if !strings.Contains(qs[1].expr, "max_over_time((irate(ifHCOutOctets{iface=") || !strings.Contains(qs[1].expr, "[60s:60s])") {
 		t.Errorf("max query: %s", qs[1].expr)
 	}
 	isProblem(t, e.do("GET", graphURL(i.ID, t0.Add(-time.Hour), t0, "&dir=sideways"), ""), 422)
@@ -408,6 +415,13 @@ func TestGraphP95IsNearestRankOverFiveMinuteAverages(t *testing.T) {
 	if *g.P95.InBPS != referenceP95(in) || *g.P95.OutBPS != referenceP95(out) {
 		t.Errorf("p95 in %v out %v, reference %v / %v", *g.P95.InBPS, *g.P95.OutBPS, referenceP95(in), referenceP95(out))
 	}
+	agg := make([]float64, len(in))
+	for k := range in {
+		agg[k] = in[k] + out[k]
+	}
+	if g.P95.AggregateBPS == nil || *g.P95.AggregateBPS != referenceP95(agg) {
+		t.Errorf("aggregate p95 %v, reference %v", g.P95.AggregateBPS, referenceP95(agg))
+	}
 	wantCov := float64(len(in)) / (24 * 12)
 	if math.Abs(g.P95.Coverage-wantCov) > 0.01 || g.P95.Coverage >= 1 {
 		t.Errorf("coverage %v, want about %v", g.P95.Coverage, wantCov)
@@ -442,3 +456,12 @@ func TestGraphErrors(t *testing.T) {
 		t.Errorf("default range: %d points", len(g.Series.In))
 	}
 }
+
+func TestGraphAggregateP95NeedsBothDirections(t *testing.T) {
+	e, i := newGraphEnv(t, &graphVM{hotVal: rising})
+	var g api.GraphResponse
+	e.getJSON(t, graphURL(i.ID, t0.Add(-2*time.Hour), t0, "&p95=true&dir=in"), &g)
+	if g.P95 == nil || g.P95.InBPS == nil || g.P95.AggregateBPS != nil {
+		t.Fatalf("one direction asked for: p95 %+v", g.P95)
+	}
+}

internal/api/graph_totals.go

@@ -0,0 +1,76 @@
+package api
+
+import "time"
+
+// GraphTotals is the data transferred over the requested range: the average-rate
+// series integrated over time. Stretches with no data add nothing, and
+// covered_seconds against range_seconds says how complete the figure is.
+type GraphTotals struct {
+	InBytes        *float64 `json:"in_bytes,omitempty" doc:"Bytes received; absent when in was not requested."`
+	OutBytes       *float64 `json:"out_bytes,omitempty" doc:"Bytes sent; absent when out was not requested."`
+	CoveredSeconds int64    `json:"covered_seconds" doc:"Seconds of the range that had data."`
+	RangeSeconds   int64    `json:"range_seconds" doc:"Length of the requested range in seconds."`
+}
+
+// unixSpan is a half-open interval in Unix seconds.
+type unixSpan struct{ lo, hi int64 }
+
+// slotSpan is the stretch of time one sample stands for, clipped to the part of the
+// range its tier owns. A raw rate is the average over the step ending at its
+// timestamp; an hourly rollup is stamped at the start of its hour. In a stitched
+// graph the hourly tier ends at the join and the raw tier starts there, so no
+// instant is counted twice.
+func slotSpan(w tierWindow, ts int64, from, to time.Time) unixSpan {
+	step := int64(w.step / time.Second)
+	s := unixSpan{lo: ts - step, hi: ts}
+	own := unixSpan{lo: from.Unix(), hi: to.Unix()}
+	if w.tier == tier1h {
+		s = unixSpan{lo: ts, hi: ts + step}
+		own.hi = min(own.hi, w.bound.Unix())
+	} else {
+		own.lo = max(own.lo, w.from.Unix()) // raw owns time from its first slot on
+	}
+	s.lo, s.hi = max(s.lo, own.lo), min(s.hi, own.hi)
+	if s.hi < s.lo {
+		s.hi = s.lo
+	}
+	return s
+}
+
+// graphTotals integrates the average series of every requested direction over
+// [from, to]. Nil when no direction has a single sample.
+func graphTotals(p *graphPlan, dirs []string, avg map[string]map[int64]float64, from, to time.Time) *GraphTotals {
+	t := &GraphTotals{RangeSeconds: to.Unix() - from.Unix()}
+	covered := map[unixSpan]struct{}{}
+	any := false
+	for _, dir := range dirs {
+		bytes := 0.0
+		for _, w := range p.tiers {
+			for _, ts := range w.slots() {
+				v, ok := avg[dir][ts]
+				if !ok {
+					continue
+				}
+				s := slotSpan(w, ts, from, to)
+				if s.hi == s.lo {
+					continue
+				}
+				any = true
+				covered[s] = struct{}{}
+				bytes += v * float64(s.hi-s.lo) / 8
+			}
+		}
+		if dir == dirIn {
+			t.InBytes = &bytes
+		} else {
+			t.OutBytes = &bytes
+		}
+	}
+	if !any {
+		return nil
+	}
+	for s := range covered {
+		t.CoveredSeconds += s.hi - s.lo
+	}
+	return t
+}

internal/api/graph_totals_test.go

@@ -0,0 +1,107 @@
+package api
+
+import (
+	"math"
+	"testing"
+	"time"
+)
+
+func totalsPlan(t *testing.T, from, to, hotStart time.Time) *graphPlan {
+	t.Helper()
+	p, err := planGraph(from, to, hotStart, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+
+// fill gives every slot of every tier the value v in both directions, except skip.
+func fill(p *graphPlan, v float64, skip func(ts int64) bool) map[string]map[int64]float64 {
+	m := map[string]map[int64]float64{dirIn: {}, dirOut: {}}
+	for _, w := range p.tiers {
+		for _, ts := range w.slots() {
+			if skip != nil && skip(ts) {
+				continue
+			}
+			m[dirIn][ts], m[dirOut][ts] = v, v/2
+		}
+	}
+	return m
+}
+
+func near(a *float64, want float64) bool { return a != nil && math.Abs(*a-want) <= want*1e-9+1e-6 }
+
+func TestGraphTotalsRawOnly(t *testing.T) {
+	now := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
+	from := now.Add(-time.Hour)
+	p := totalsPlan(t, from, now, now.Add(-30*24*time.Hour))
+	got := graphTotals(p, []string{dirIn, dirOut}, fill(p, 8000, nil), from, now)
+	// 8 kbit/s for 3600 s is 3.6 MB; out is half.
+	if got == nil || !near(got.InBytes, 3_600_000) || !near(got.OutBytes, 1_800_000) {
+		t.Fatalf("totals %+v", got)
+	}
+	if got.CoveredSeconds != 3600 || got.RangeSeconds != 3600 {
+		t.Errorf("covered %d range %d", got.CoveredSeconds, got.RangeSeconds)
+	}
+}
+
+func TestGraphTotalsUnalignedRangeIsClipped(t *testing.T) {
+	now := time.Date(2026, 10, 22, 12, 0, 30, 0, time.UTC)
+	from := now.Add(-time.Hour)
+	p := totalsPlan(t, from, now, now.Add(-30*24*time.Hour))
+	got := graphTotals(p, []string{dirIn}, fill(p, 8000, nil), from, now)
+	if got == nil || got.OutBytes != nil || got.InBytes == nil || *got.InBytes > 3_600_000+1 {
+		t.Fatalf("totals %+v", got)
+	}
+}
+
+func TestGraphTotalsStitchedNoDoubleCount(t *testing.T) {
+	now := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
+	hotStart := now.Add(-30 * 24 * time.Hour)
+	from := now.Add(-40 * 24 * time.Hour)
+	p := totalsPlan(t, from, now, hotStart)
+	if p.stitchedAt == nil {
+		t.Fatal("not stitched")
+	}
+	got := graphTotals(p, []string{dirIn}, fill(p, 8000, nil), from, now)
+	// 8 kbit/s is 1000 B/s: the total must be exactly that over what was covered, so nothing is counted twice.
+	if got == nil || !near(got.InBytes, 1000*float64(got.CoveredSeconds)) {
+		t.Fatalf("totals %+v", got)
+	}
+	// Only the partial step at each end (aligned away) may be missing.
+	if lost := got.RangeSeconds - got.CoveredSeconds; lost < 0 || lost > int64(2*p.newest().step/time.Second) {
+		t.Errorf("covered %d of %d", got.CoveredSeconds, got.RangeSeconds)
+	}
+}
+
+func TestGraphTotalsGapsContributeNothing(t *testing.T) {
+	now := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
+	from := now.Add(-time.Hour)
+	p := totalsPlan(t, from, now, now.Add(-30*24*time.Hour))
+	half := now.Add(-30 * time.Minute).Unix()
+	got := graphTotals(p, []string{dirIn}, fill(p, 8000, func(ts int64) bool { return ts > half }), from, now)
+	if got == nil || !near(got.InBytes, 1_800_000) || got.CoveredSeconds != 1800 {
+		t.Fatalf("totals %+v", got)
+	}
+	if graphTotals(p, []string{dirIn}, fill(p, 8000, func(int64) bool { return true }), from, now) != nil {
+		t.Error("no data should give nil totals")
+	}
+}
+
+func TestAggregateP95SumsOnlySlotsWithBothDirections(t *testing.T) {
+	slots := map[string]map[int64]float64{
+		dirIn:  {300: 10, 600: 20, 900: 30, 1200: 1000},
+		dirOut: {300: 1, 600: 2, 900: 3},
+	}
+	got := aggregateP95(slots)
+	// slot 1200 has no out sample and is ignored: sums are 11, 22, 33 and the nearest-rank 95th of three is the top.
+	if !near(got, 33) {
+		t.Fatalf("aggregate %v, want 33", got)
+	}
+	if aggregateP95(map[string]map[int64]float64{dirIn: {300: 1}}) != nil {
+		t.Error("one direction only gives no aggregate")
+	}
+	if aggregateP95(map[string]map[int64]float64{dirIn: {300: 1}, dirOut: {600: 1}}) != nil {
+		t.Error("no shared slot gives no aggregate")
+	}
+}

internal/api/graph_types.go

@@ -102,26 +102,29 @@ func (g *GraphSeries) set(dir string, isMax bool, pts []Point) {
 
 // GraphP95 is the 95th percentile of 5-minute averages (nearest rank).
 type GraphP95 struct {
-	InBPS    *float64 `json:"in_bps" nullable:"true"`
-	OutBPS   *float64 `json:"out_bps" nullable:"true"`
-	Source   string   `json:"source" enum:"materialised,live"`
-	Coverage float64  `json:"coverage" doc:"Share of the drawn range's 5-minute slots that had data, 0 to 1."`
-	Month    string   `json:"month,omitempty" doc:"YYYY-MM when the figure is for one calendar month of the site's timezone."`
-	Partial  bool     `json:"partial" doc:"Coverage is below 90%: show it next to the figure."`
+	InBPS  *float64 `json:"in_bps" nullable:"true"`
+	OutBPS *float64 `json:"out_bps" nullable:"true"`
+	// AggregateBPS is the 95th percentile of in+out per 5-minute slot (LibreNMS's aggregate), over the raw part of the range.
+	AggregateBPS *float64 `json:"aggregate_bps,omitempty" doc:"95th percentile of in+out per 5-minute slot, slots with both directions only; absent when one direction was not asked for, or the range has no raw data."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
+	Source       string   `json:"source" enum:"materialised,live"`
+	Coverage     float64  `json:"coverage" doc:"Share of the drawn range's 5-minute slots that had data, 0 to 1."`
+	Month        string   `json:"month,omitempty" doc:"YYYY-MM when the figure is for one calendar month of the site's timezone."`
+	Partial      bool     `json:"partial" doc:"Coverage is below 90%: show it next to the figure."`
 }
 
 // GraphResponse is a rate graph for one interface. A graph answered from two
 // stores always carries stitched_at; stretches with no data in either store are
 // listed in partial; an unreliable interface has no series at all.
 type GraphResponse struct {
-	Iface            GraphIface  `json:"iface"`
-	StepSeconds      int64       `json:"step_seconds" doc:"Step of the newest tier."`
-	Tier             string      `json:"tier" enum:"raw,1h" doc:"Tier of the newest data."`
-	StitchedAt       *time.Time  `json:"stitched_at,omitempty" doc:"Where the hourly long-store data ends and raw hot data begins."`
-	Tiers            []GraphTier `json:"tiers,omitempty" doc:"Present when two stores answered."`
-	UnreliableReason string      `json:"unreliable_reason,omitempty" doc:"Why the counters are not trusted, e.g. counter_32bit_fast_link."`
-	Partial          []TimeSpan  `json:"partial" doc:"Stretches of the range where neither store has data."`
-	Unreliable       bool        `json:"unreliable"`
-	Series           GraphSeries `json:"series"`
-	P95              *GraphP95   `json:"p95,omitempty"`
+	Iface            GraphIface   `json:"iface"`
+	StepSeconds      int64        `json:"step_seconds" doc:"Step of the newest tier."`
+	Tier             string       `json:"tier" enum:"raw,1h" doc:"Tier of the newest data."`
+	StitchedAt       *time.Time   `json:"stitched_at,omitempty" doc:"Where the hourly long-store data ends and raw hot data begins."`
+	Tiers            []GraphTier  `json:"tiers,omitempty" doc:"Present when two stores answered."`
+	UnreliableReason string       `json:"unreliable_reason,omitempty" doc:"Why the counters are not trusted, e.g. counter_32bit_fast_link."`
+	Partial          []TimeSpan   `json:"partial" doc:"Stretches of the range where neither store has data."`
+	Unreliable       bool         `json:"unreliable"`
+	Series           GraphSeries  `json:"series"`
+	P95              *GraphP95    `json:"p95,omitempty"`
+	Totals           *GraphTotals `json:"totals,omitempty" doc:"Bytes transferred over the range; absent when unreliable or without data."`
 }

internal/api/notify.go

@@ -0,0 +1,125 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strings"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/notify"
+)
+
+// NotifyTester sends one immediate test notification through the running outbound policy. *notify.Queue fits.
+type NotifyTester interface {
+	SendTest(ctx context.Context, r notify.Receiver, state string) (notify.TestResult, error)
+}
+
+// NotifyRequest is a receiver as the settings form holds it plus the sample state to render.
+type NotifyRequest struct {
+	Receiver ReceiverForm `json:"receiver" doc:"The receiver as edited. Blank credentials resolve to the stored receiver of the same name."`
+	State    string       `json:"state,omitempty" enum:"firing,resolved,flapping,stale,suppressed_summary" doc:"Sample state; default firing."`
+}
+
+// NotifyPreview is a rendered sample payload, or why it could not be rendered.
+type NotifyPreview struct {
+	Output string `json:"output" doc:"The payload that would be sent, redacted. Empty on error."`
+	Error  string `json:"error" doc:"Template, JSON or receiver validation problem. Empty on success."`
+}
+
+// NotifyTestResult is the outcome of one test send; the receiver's own failures are reported here with a 200.
+type NotifyTestResult struct {
+	Payload      string `json:"payload" doc:"What was sent, redacted."`
+	StatusCode   int    `json:"status_code" doc:"HTTP status of a webhook; 0 for SMTP or when no response arrived."`
+	ResponseBody string `json:"response_body" doc:"First 4 KB of the response, redacted."`
+	DurationMS   int64  `json:"duration_ms"`
+	Error        string `json:"error" doc:"Why the send failed. Empty on success."`
+	OK           bool   `json:"ok"`
+}
+
+// NotifyPreset is a built-in webhook template.
+type NotifyPreset struct {
+	ID          string `json:"id"`
+	Name        string `json:"name"`
+	Description string `json:"description"`
+	Template    string `json:"template"`
+	URLHint     string `json:"url_hint" doc:"Shape of the service's webhook URL, for a placeholder."`
+	ContentType string `json:"content_type"`
+}
+
+func (s *server) registerNotify(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "list-notify-presets", Method: http.MethodGet, Path: "/api/v1/notify/presets", Tags: []string{"notify"},
+		Summary: "Built-in webhook templates", Description: "Teams (Workflows), Slack, Discord and generic JSON.",
+	}, s.listNotifyPresets)
+	huma.Register(api, huma.Operation{
+		OperationID: "preview-notify", Method: http.MethodPost, Path: "/api/v1/notify/preview", Tags: []string{"notify"},
+		Summary: "Render a receiver's template", Description: "Admin only. Renders the template against a sample " +
+			"notification with the delivery code path. Nothing is sent or stored.",
+	}, s.previewNotify)
+	huma.Register(api, huma.Operation{
+		OperationID: "test-notify", Method: http.MethodPost, Path: "/api/v1/notify/test", Tags: []string{"notify"},
+		Summary: "Send one test notification",
+		Description: "Admin only. Sends a sample notification to the receiver once, now, through the outbound allow-list, " +
+			"without the retry queue. Only an audit row is stored. At most two run at a time.",
+		Errors: []int{http.StatusTooManyRequests, http.StatusServiceUnavailable},
+	}, s.testNotify)
+}
+
+func (s *server) listNotifyPresets(context.Context, *struct{}) (*struct{ Body []NotifyPreset }, error) {
+	ps := notify.Presets()
+	out := make([]NotifyPreset, len(ps))
+	for i, p := range ps {
+		out[i] = NotifyPreset(p)
+	}
+	return &struct{ Body []NotifyPreset }{out}, nil
+}
+
+// stateOrDefault is the requested sample state, firing when unset.
+func stateOrDefault(st string) string {
+	if st == "" {
+		return notify.StateFiring
+	}
+	return st
+}
+
+// resolveReceiver builds the receiver to use from the form, filling blank credentials from the stored receiver
+// of the same name, exactly like PUT /settings/form.
+func (s *server) resolveReceiver(f ReceiverForm) notify.Receiver {
+	var old notify.Receiver
+	if a, ok := s.Settings.(SettingsConfigAdmin); ok {
+		cfg, _ := a.CurrentConfig()
+		old = storedReceivers(cfg.Alerting)[strings.TrimSpace(f.Name)]
+	}
+	return receiverFrom(f, old)
+}
+
+func (s *server) previewNotify(_ context.Context, in *struct{ Body NotifyRequest }) (*struct{ Body NotifyPreview }, error) {
+	r := receiverFrom(in.Body.Receiver, notify.Receiver{})
+	out, err := notify.Preview(r, stateOrDefault(in.Body.State), s.Clock.Now())
+	p := NotifyPreview{Output: out}
+	if err != nil {
+		p.Error = err.Error()
+	}
+	return &struct{ Body NotifyPreview }{p}, nil
+}
+
+func (s *server) testNotify(ctx context.Context, in *struct{ Body NotifyRequest }) (*struct{ Body NotifyTestResult }, error) {
+	if s.Notify == nil {
+		return nil, errNotConfigured("notifications")
+	}
+	r, state := s.resolveReceiver(in.Body.Receiver), stateOrDefault(in.Body.State)
+	res, err := s.Notify.SendTest(ctx, r, state)
+	if errors.Is(err, notify.ErrTestBusy) {
+		return nil, huma.Error429TooManyRequests("another test send is in progress; try again in a moment")
+	}
+	if err != nil {
+		return nil, huma.Error422UnprocessableEntity(err.Error())
+	}
+	detail := map[string]any{"kind": r.Kind, "state": state, "ok": res.OK, "status_code": res.StatusCode}
+	if aerr := s.Store.AppendAudit(ctx, actorOf(ctx), "notify.test", "receiver:"+r.Name, nil, detail); aerr != nil {
+		s.Logger.Warn("audit notify.test", "err", aerr)
+	}
+	return &struct{ Body NotifyTestResult }{NotifyTestResult(res)}, nil
+}

internal/api/notify_test.go

@@ -0,0 +1,77 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/auth"
+	"github.com/v10networks/nmon/internal/auth/authtest"
+	"github.com/v10networks/nmon/internal/config"
+	"github.com/v10networks/nmon/internal/notify"
+)
+
+type fakeNotify struct {
+	got   notify.Receiver
+	state string
+	err   error
+}
+
+func (f *fakeNotify) SendTest(_ context.Context, r notify.Receiver, state string) (notify.TestResult, error) {
+	f.got, f.state = r, state
+	return notify.TestResult{Payload: "{}", StatusCode: 200, OK: true}, f.err
+}
+
+func TestNotifyPresetsPreviewAndTestSend(t *testing.T) {
+	fs := &formSettings{cfg: config.Default()}
+	fn := &fakeNotify{}
+	e := newAuthedEnv(t, func(d *api.Deps) { d.Settings = fs; d.Notify = fn })
+	admin, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
+	viewer, _ := authtest.Mint(t, e.auth, auth.RoleViewer)
+
+	var presets []api.NotifyPreset
+	rec := e.call(http.MethodGet, "/api/v1/notify/presets", "", viewer)
+	if err := json.Unmarshal(rec.Body.Bytes(), &presets); err != nil || len(presets) != 4 || presets[1].ID != "teams" {
+		t.Fatalf("presets: %d %s", rec.Code, rec.Body)
+	}
+	body := func(tpl string) string {
+		b, _ := json.Marshal(map[string]any{"state": "resolved", "receiver": map[string]any{
+			"name": "teams", "kind": "webhook", "url": "https://x.example/", "template": tpl, "headers": map[string]string{"Authorization": ""}, "smtp": map[string]any{"clear_password": false, "from": "", "host": "", "password": "", "port": 0,
+				"starttls": false, "to": []string{}, "username": ""}}})
+		return string(b)
+	}
+	for _, path := range []string{"/api/v1/notify/preview", "/api/v1/notify/test"} {
+		if code := e.call(http.MethodPost, path, body(presets[1].Template), viewer).Code; code != http.StatusForbidden {
+			t.Fatalf("viewer POST %s: %d, want 403", path, code)
+		}
+	}
+	var pv api.NotifyPreview
+	rec = e.call(http.MethodPost, "/api/v1/notify/preview", body(presets[1].Template), admin)
+	if err := json.Unmarshal(rec.Body.Bytes(), &pv); err != nil || pv.Error != "" || !strings.Contains(pv.Output, "AdaptiveCard") {
+		t.Fatalf("preview: %d %s", rec.Code, rec.Body)
+	}
+	rec = e.call(http.MethodPost, "/api/v1/notify/preview", body("{{.Nope}}"), admin)
+	if err := json.Unmarshal(rec.Body.Bytes(), &pv); err != nil || rec.Code != http.StatusOK || pv.Error == "" || pv.Output != "" {
+		t.Fatalf("broken template: %d %s", rec.Code, rec.Body)
+	}
+
+	// Blank header value resolves to the stored receiver's secret.
+	fs.cfg.Alerting.Receivers = nil
+	put := `{"receivers":[{"name":"teams","kind":"webhook","url":"https://x.example/","headers":{"Authorization":"Bearer s3cret"},"template":"","smtp":{"clear_password":false,"from":"","host":"","password":"","port":0,"starttls":false,"to":[],"username":""}}]}`
+	if rec := e.call(http.MethodPut, "/api/v1/settings/form", put, admin); rec.Code != http.StatusOK {
+		t.Fatalf("seed receiver: %d %s", rec.Code, rec.Body)
+	}
+	rec = e.call(http.MethodPost, "/api/v1/notify/test", body(presets[1].Template), admin)
+	var tr api.NotifyTestResult
+	if err := json.Unmarshal(rec.Body.Bytes(), &tr); err != nil || !tr.OK || fn.state != "resolved" ||
+		fn.got.Headers["Authorization"] != "Bearer s3cret" || strings.Contains(rec.Body.String(), "s3cret") {
+		t.Fatalf("test: %d %s got=%+v", rec.Code, rec.Body, fn.got)
+	}
+	fn.err = notify.ErrTestBusy
+	if code := e.call(http.MethodPost, "/api/v1/notify/test", body(""), admin).Code; code != http.StatusTooManyRequests {
+		t.Fatalf("busy: %d", code)
+	}
+}

internal/api/overview.go

@@ -88,7 +88,7 @@ func (s *server) overviewMetrics(ctx context.Context, resp *OverviewResponse, d
 		if len(hc)+len(soft) == 0 {
 			break
 		}
-		vals, err := s.fetchMetricSeries(ctx, trafficExpr(hc, soft, dir, step), plan)
+		vals, err := s.fetchMetricSeries(ctx, trafficExpr(hc, soft, dir, step, s.pollEvery()), plan)
 		if err != nil {
 			resp.Partial = append(resp.Partial, "traffic")
 			resp.Traffic.In, resp.Traffic.Out = []Point{}, []Point{}
@@ -127,9 +127,9 @@ func countedIfaces(ifs []store.Interface, t *OverviewTraffic) (hc, soft []string
 }
 
 // trafficExpr is the summed bits per second of dir; HC counters where a port has them, else the 32-bit ones.
-func trafficExpr(hc, soft []string, dir string, step time.Duration) string {
+func trafficExpr(hc, soft []string, dir string, step, every time.Duration) string {
 	names := map[string][2]string{dirIn: {"ifHCInOctets", "ifInOctets"}, dirOut: {"ifHCOutOctets", "ifOutOctets"}}[dir]
-	win := secs(max(step, minRateWindow))
+	win := secs(max(step, rateWindowFor(every)))
 	var parts []string
 	for n, ids := range [][]string{hc, soft} {
 		if len(ids) > 0 {

internal/api/policy.go

@@ -39,10 +39,25 @@ func Routes() []Route {
 }
 
 // adminReads are GET routes only an admin may call: they reveal who can reach nmon or how it is configured.
-var adminReads = []string{"GET /api/v1/tokens", "GET /api/v1/settings", "GET /api/v1/audit"}
+var adminReads = []string{"GET /api/v1/tokens", "GET /api/v1/settings", "GET /api/v1/settings/form", "GET /api/v1/audit",
+	"GET /api/v1/devices/{slug}/snmp-debug"}
 
 // viewerPosts are non-GET routes that only read, so a viewer may call them (a query endpoint taking a body).
-var viewerPosts = []string{"POST /api/v1/alert-policies/preview"}
+var viewerPosts = []string{"POST /api/v1/alert-policies/preview", "POST /api/v1/alert-rules/test"}
+
+// installRoutes are the only API routes served in install mode (the first-install wizard, before any sign-in exists):
+// the setup routes, the typed settings form, webhook presets/preview/test, adding and watching devices, alert
+// policies, the profile list, secrets and meta. Everything else answers 503 "setup not finished".
+var installRoutes = []string{
+	"GET /api/v1/setup", "POST /api/v1/setup/signin", "POST /api/v1/setup/recommended-policies", "POST /api/v1/setup/complete",
+	"GET /api/v1/meta", "GET /api/v1/settings/form", "PUT /api/v1/settings/form",
+	"GET /api/v1/notify/presets", "POST /api/v1/notify/preview", "POST /api/v1/notify/test",
+	"GET /api/v1/devices", "POST /api/v1/devices", "POST /api/v1/devices/import", "GET /api/v1/devices/{slug}",
+	"GET /api/v1/alert-policies", "POST /api/v1/alert-policies", "GET /api/v1/profiles", "PUT /api/v1/secrets/{name}",
+}
+
+// InstallRoutes lists the routes install mode serves, for the tests that keep it in step with the API.
+func InstallRoutes() []string { return append([]string(nil), installRoutes...) }
 
 // DeclarePolicy states the minimum role of each route that departs from the default (GET and HEAD need viewer,
 // everything else admin).
@@ -53,6 +68,9 @@ func DeclarePolicy(p *auth.Policy) {
 	for _, pat := range viewerPosts {
 		p.Require(pat, auth.RoleViewer)
 	}
+	// The setup state is public: the UI asks it before anyone can sign in, to learn whether to show the wizard.
+	p.Public("GET /api/v1/setup")
+	p.AllowInstall(installRoutes...)
 }
 
 // ReadOnlyPosts are the non-GET patterns declared viewer-level, for the viewer-cannot-mutate test to skip.

internal/api/poll_tools.go

@@ -0,0 +1,155 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"time"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/poller"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// PollTools is what the poll test, poll-now and SNMP debug routes need from the poller. *poller.Poller implements it.
+type PollTools interface {
+	PollNow(id int64) bool
+	TestPoll(ctx context.Context, id int64, group time.Duration) (*poller.Trace, error)
+	SetDebug(id int64, minutes int) poller.DebugState
+	DebugState(id int64) poller.DebugState
+}
+
+// PollNowView answers POST /devices/{slug}/poll-now.
+type PollNowView struct {
+	Queued bool `json:"queued" doc:"The device's primary group will be polled right away; its result shows in the normal places."`
+}
+
+// SNMPDebugBody is the body of PUT /devices/{slug}/snmp-debug.
+type SNMPDebugBody struct {
+	Minutes int `json:"minutes" minimum:"0" maximum:"60" doc:"Minutes to record scheduled cycles for; 0 turns recording off."`
+}
+
+func (s *server) registerPollTools(api huma.API) {
+	tags := []string{"devices"}
+	huma.Register(api, huma.Operation{
+		OperationID: "poll-device-now", Method: http.MethodPost, Path: "/api/v1/devices/{slug}/poll-now", Tags: tags,
+		Summary: "Poll a device now", DefaultStatus: http.StatusAccepted,
+		Description: "Asks the poller to run the device's primary group right away, outside its schedule. The result is recorded " +
+			"like any poll. Admin-only.",
+		Errors: []int{http.StatusNotFound, http.StatusConflict, http.StatusServiceUnavailable},
+	}, s.pollNow)
+	huma.Register(api, huma.Operation{
+		OperationID: "test-poll-device", Method: http.MethodPost, Path: "/api/v1/devices/{slug}/poll-test", Tags: tags,
+		Summary: "Test-poll a device with a trace",
+		Description: "Runs one synchronous SNMP cycle of the device's real poll plan (stored credential, resolved profile, address) " +
+			"and returns a trace: per-request timings and errors, and every requested OID with its result. Nothing is written to " +
+			"the metric store and no poller state changes. One test per device at a time (409), a few at once overall (429), " +
+			"60 s at most, 5,000 OID rows at most. Credentials are never returned. Admin-only.",
+		Errors: []int{http.StatusNotFound, http.StatusConflict, http.StatusUnprocessableEntity, http.StatusTooManyRequests,
+			http.StatusServiceUnavailable},
+	}, s.testPoll)
+	huma.Register(api, huma.Operation{
+		OperationID: "get-snmp-debug", Method: http.MethodGet, Path: "/api/v1/devices/{slug}/snmp-debug", Tags: tags,
+		Summary: "Read a device's SNMP debug recording",
+		Description: "Whether recording is on and the last cycles recorded (20 at most, newest first). In memory only: cleared " +
+			"on restart. Admin-only.",
+		Errors: []int{http.StatusNotFound, http.StatusServiceUnavailable},
+	}, s.getSNMPDebug)
+	huma.Register(api, huma.Operation{
+		OperationID: "set-snmp-debug", Method: http.MethodPut, Path: "/api/v1/devices/{slug}/snmp-debug", Tags: tags,
+		Summary: "Switch a device's SNMP debug recording on or off",
+		Description: "While on, the poller records each scheduled cycle of the device (per-request timings and errors, per-OID " +
+			"results) and logs a one-line summary at INFO. It stops by itself after the given minutes (60 at most). Starting " +
+			"a recording drops the previous one. Admin-only.",
+		Errors: []int{http.StatusNotFound, http.StatusConflict, http.StatusServiceUnavailable},
+	}, s.setSNMPDebug)
+}
+
+type slugIn struct {
+	Slug string `path:"slug" doc:"Device slug."`
+}
+
+// pollableDevice loads the device and refuses one that is gone, retired or has no SNMP credential.
+func (s *server) pollableDevice(ctx context.Context, slug string) (*store.Device, error) {
+	if s.PollTools == nil {
+		return nil, errNotConfigured("the poller")
+	}
+	d, err := s.Store.GetDeviceBySlug(ctx, slug)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	if d.PurgedAt != nil {
+		return nil, huma.Error404NotFound("not found")
+	}
+	if d.RetiredAt != nil || d.Kind != store.KindNetwork || d.SNMPCredentialID == nil {
+		return nil, huma.Error409Conflict("device is retired or has no SNMP credential; it is not polled over SNMP")
+	}
+	return d, nil
+}
+
+func (s *server) pollNow(ctx context.Context, in *slugIn) (*struct{ Body PollNowView }, error) {
+	d, err := s.pollableDevice(ctx, in.Slug)
+	if err != nil {
+		return nil, err
+	}
+	if !d.Enabled || !s.PollTools.PollNow(d.ID) {
+		return nil, huma.Error409Conflict("device is not being polled (disabled, or the poller has not loaded it yet)")
+	}
+	s.auditPoll(ctx, "device.poll_now", d.Slug, nil)
+	return &struct{ Body PollNowView }{PollNowView{Queued: true}}, nil
+}
+
+func (s *server) testPoll(ctx context.Context, in *struct {
+	Slug     string `path:"slug" doc:"Device slug."`
+	Interval int    `query:"interval_seconds" minimum:"0" doc:"Test only the group with this cadence in seconds; 0 tests every group."`
+}) (*struct{ Body *poller.Trace }, error) {
+	d, err := s.pollableDevice(ctx, in.Slug)
+	if err != nil {
+		return nil, err
+	}
+	tr, err := s.PollTools.TestPoll(ctx, d.ID, time.Duration(in.Interval)*time.Second)
+	switch {
+	case errors.Is(err, poller.ErrTestBusy):
+		return nil, huma.Error409Conflict("a test poll of this device is already running")
+	case errors.Is(err, poller.ErrTestLimit):
+		return nil, huma.Error429TooManyRequests("too many test polls are running; try again shortly")
+	case errors.Is(err, poller.ErrNotPolled), errors.Is(err, poller.ErrNoSuchPoll):
+		return nil, huma.Error422UnprocessableEntity(err.Error())
+	case err != nil:
+		return nil, s.storeErr(err)
+	}
+	s.auditPoll(ctx, "device.poll_test", d.Slug, map[string]any{
+		"interval_seconds": in.Interval, "requested": tr.Requested, "returned": tr.Returned, "missing": tr.Missing,
+		"duration_ms": tr.DurationMS,
+	})
+	return &struct{ Body *poller.Trace }{tr}, nil
+}
+
+func (s *server) getSNMPDebug(ctx context.Context, in *slugIn) (*struct{ Body poller.DebugState }, error) {
+	d, err := s.pollableDevice(ctx, in.Slug)
+	if err != nil {
+		return nil, err
+	}
+	return &struct{ Body poller.DebugState }{s.PollTools.DebugState(d.ID)}, nil
+}
+
+func (s *server) setSNMPDebug(ctx context.Context, in *struct {
+	Slug string `path:"slug" doc:"Device slug."`
+	Body SNMPDebugBody
+}) (*struct{ Body poller.DebugState }, error) {
+	d, err := s.pollableDevice(ctx, in.Slug)
+	if err != nil {
+		return nil, err
+	}
+	st := s.PollTools.SetDebug(d.ID, in.Body.Minutes)
+	s.auditPoll(ctx, "device.snmp_debug", d.Slug, map[string]any{"minutes": in.Body.Minutes})
+	return &struct{ Body poller.DebugState }{st}, nil
+}
+
+// auditPoll records an operator action on a device's polling. A failed audit write is logged, not fatal: the action ran.
+func (s *server) auditPoll(ctx context.Context, action, slug string, detail map[string]any) {
+	if err := s.Store.AppendAudit(ctx, actorOf(ctx), action, slug, nil, detail); err != nil {
+		s.Logger.Warn("audit "+action, "device", slug, "err", err)
+	}
+}

internal/api/poll_tools_test.go

@@ -0,0 +1,123 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/fixturegen"
+	"github.com/v10networks/nmon/internal/poller"
+	"github.com/v10networks/nmon/internal/simagent"
+	"github.com/v10networks/nmon/internal/testutil"
+)
+
+// fakeTools stands in for the poller.
+type fakeTools struct {
+	mu      sync.Mutex
+	polled  bool
+	nowIDs  []int64
+	testErr error
+	minutes int
+}
+
+func (f *fakeTools) PollNow(id int64) bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.nowIDs = append(f.nowIDs, id)
+	return f.polled
+}
+
+func (f *fakeTools) TestPoll(_ context.Context, _ int64, group time.Duration) (*poller.Trace, error) {
+	if f.testErr != nil {
+		return nil, f.testErr
+	}
+	return &poller.Trace{Device: "x", Target: "[::1]:161", SNMPVersion: "2c", Requested: 3, Returned: 2, Missing: 1,
+		Groups: []poller.GroupTrace{{IntervalSeconds: int(group / time.Second), Requests: []poller.RequestTrace{}, Results: []poller.OIDResult{}}}}, nil
+}
+
+func (f *fakeTools) SetDebug(_ int64, minutes int) poller.DebugState {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.minutes = minutes
+	return poller.DebugState{Enabled: minutes > 0, MaxCycles: poller.DebugCycles, Cycles: []poller.GroupTrace{}}
+}
+
+func (f *fakeTools) DebugState(int64) poller.DebugState {
+	return poller.DebugState{MaxCycles: poller.DebugCycles, Cycles: []poller.GroupTrace{}}
+}
+
+func newToolsEnv(t *testing.T, f *fakeTools) (*env, string) {
+	a := testutil.StartAgent(t, simagent.Config{
+		Recording: fixturegen.Generic(fixturegen.Options{Ports: 2, SysName: "d1", HCCounters: true}), Community: community,
+	})
+	e := newEnvDeps(t, int(a.Addr().Port()), func(d *api.Deps) { d.PollTools = f })
+	body := fmt.Sprintf(`{"name":"D One","addresses":["::1"],"snmp":{"version":"v2c","community":%q}}`, community)
+	rec := e.do("POST", "/api/v1/devices", body)
+	if rec.Code != http.StatusCreated {
+		t.Fatalf("POST: %d %s", rec.Code, rec.Body)
+	}
+	var dev api.DeviceView
+	_ = json.Unmarshal(rec.Body.Bytes(), &dev)
+	return e, "/api/v1/devices/" + dev.Slug
+}
+
+func TestPollNowQueuesOrExplainsWhyNot(t *testing.T) {
+	f := &fakeTools{polled: true}
+	e, base := newToolsEnv(t, f)
+	if rec := e.do("POST", base+"/poll-now", ""); rec.Code != http.StatusAccepted || !strings.Contains(rec.Body.String(), `"queued":true`) {
+		t.Fatalf("poll-now: %d %s", rec.Code, rec.Body)
+	}
+	f.polled = false
+	isProblem(t, e.do("POST", base+"/poll-now", ""), http.StatusConflict)
+	isProblem(t, e.do("POST", "/api/v1/devices/nope/poll-now", ""), http.StatusNotFound)
+}
+
+func TestPollTestReturnsTraceAuditsAndMapsLimits(t *testing.T) {
+	f := &fakeTools{}
+	e, base := newToolsEnv(t, f)
+	rec := e.do("POST", base+"/poll-test?interval_seconds=300", "")
+	if rec.Code != http.StatusOK {
+		t.Fatalf("poll-test: %d %s", rec.Code, rec.Body)
+	}
+	var tr poller.Trace
+	if err := json.Unmarshal(rec.Body.Bytes(), &tr); err != nil || tr.Requested != 3 || tr.Groups[0].IntervalSeconds != 300 {
+		t.Fatalf("trace %+v (%v)", tr, err)
+	}
+	if a := e.do("GET", "/api/v1/audit", ""); !strings.Contains(a.Body.String(), "device.poll_test") {
+		t.Errorf("no audit entry: %s", a.Body)
+	}
+	for err, status := range map[error]int{
+		poller.ErrTestBusy:   http.StatusConflict,
+		poller.ErrTestLimit:  http.StatusTooManyRequests,
+		poller.ErrNotPolled:  http.StatusUnprocessableEntity,
+		poller.ErrNoSuchPoll: http.StatusUnprocessableEntity,
+	} {
+		f.testErr = err
+		isProblem(t, e.do("POST", base+"/poll-test", ""), status)
+	}
+}
+
+func TestSNMPDebugSwitchesAndIsBounded(t *testing.T) {
+	f := &fakeTools{}
+	e, base := newToolsEnv(t, f)
+	rec := e.do("PUT", base+"/snmp-debug", `{"minutes":15}`)
+	if rec.Code != http.StatusOK || f.minutes != 15 || !strings.Contains(rec.Body.String(), `"enabled":true`) {
+		t.Fatalf("PUT: %d %s (minutes %d)", rec.Code, rec.Body, f.minutes)
+	}
+	if rec := e.do("GET", base+"/snmp-debug", ""); rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), `"cycles":[]`) {
+		t.Fatalf("GET: %d %s", rec.Code, rec.Body)
+	}
+	isProblem(t, e.do("PUT", base+"/snmp-debug", `{"minutes":61}`), http.StatusUnprocessableEntity)
+	isProblem(t, e.do("PUT", base+"/snmp-debug", `{"minutes":-1}`), http.StatusUnprocessableEntity)
+}
+
+func TestPollToolsAnswer503WithoutAPoller(t *testing.T) {
+	e := newEnv(t, 161)
+	isProblem(t, e.do("POST", "/api/v1/devices/x/poll-now", ""), http.StatusServiceUnavailable)
+}

internal/api/rates.go

@@ -3,12 +3,14 @@ package api
 import (
 	"context"
 	"strings"
+	"time"
 
 	"github.com/v10networks/nmon/internal/store"
 )
 
-// liveWindow is the lookback of the "live" rates on device and search pages.
-const liveWindow = "5m"
+// liveWindow is the lookback of the "live" rates on device and search pages: two default poll intervals, so a
+// device polled at the default always has two samples inside it (never under the five minutes it used to be).
+func (s *server) liveWindow() string { return secs(rateWindowFor(s.pollEvery())) }
 
 // ifRates holds the five-minute rates of one interface by the counter they came
 // from; pick chooses between the 64-bit and 32-bit counters.
@@ -29,17 +31,17 @@ var rateKinds = []struct {
 // ratesExpr is one MetricsQL expression yielding every live rate of the interfaces
 // matched by match (label matchers), each series tagged with k. When uptime is set
 // it also yields the device's last sysUpTime under k="uptime".
-func ratesExpr(match string, uptime bool) string {
+func ratesExpr(match string, uptime bool, window string) string {
 	parts := make([]string, 0, len(rateKinds)+1)
 	for _, k := range rateKinds {
 		scale := ""
 		if k.bits {
 			scale = "*8"
 		}
-		parts = append(parts, `label_set(rate(`+k.metric+`{`+match+`}[`+liveWindow+`])`+scale+`, "k", `+quote(k.k)+`)`)
+		parts = append(parts, `label_set(rate(`+k.metric+`{`+match+`}[`+window+`])`+scale+`, "k", `+quote(k.k)+`)`)
 	}
 	if uptime {
-		parts = append(parts, `label_set(last_over_time(sysUpTime{`+match+`}[10m]), "k", "uptime")`)
+		parts = append(parts, `label_set(last_over_time(sysUpTime{`+match+`}[`+window+`]), "k", "uptime")`)
 	}
 	return strings.Join(parts, " or ")
 }
@@ -54,7 +56,7 @@ func (s *server) liveRates(ctx context.Context, match string, uptime bool) (map[
 	defer release()
 	qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
 	defer cancel()
-	body, err := s.Querier.Query(qctx, "hot", ratesExpr(match, uptime), s.Clock.Now())
+	body, err := s.Querier.Query(qctx, "hot", ratesExpr(match, uptime, s.liveWindow()), s.Clock.Now())
 	if err != nil {
 		s.Logger.Warn("live rates unavailable", "err", err)
 		return nil, nil
@@ -123,3 +125,6 @@ func (x *ifRates) errs(i *store.Interface) (in, out *float64) {
 	}
 	return x.inErr, x.outErr
 }
+
+// pollEvery is the default poll interval right now: the live cadence when the app gave one, else the fixed option.
+func (s *server) pollEvery() time.Duration { return s.Cadence.Or(s.PollInterval) }

internal/api/reach.go

@@ -30,11 +30,13 @@ type ReachabilitySource interface {
 type persistedReachability struct {
 	q        Querier
 	clk      clock.Clock
+	cadence  *store.PollCadence
 	interval time.Duration
 }
 
 // upLookback is how stale an up series may be and still count.
-const upLookback = "5m"
+// It must outlast two default SNMP polls, or nmon_snmp_up would drop out between polls.
+const upLookback = "11m"
 
 // maxScoped is the most devices a query names explicitly; beyond it all devices are fetched.
 const maxScoped = 200
@@ -45,7 +47,7 @@ func (p *persistedReachability) Reachability(ctx context.Context, devs []store.D
 	out := make(map[string]string, len(devs))
 	for i := range devs {
 		d := &devs[i]
-		out[d.Slug] = reachabilityOf(d, icmp[d.Slug], snmp[d.Slug], now, p.interval)
+		out[d.Slug] = reachabilityOf(d, icmp[d.Slug], snmp[d.Slug], now, d.PollInterval(p.cadence.Or(p.interval)))
 	}
 	return out
 }

internal/api/settings_form.go

@@ -0,0 +1,102 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/config"
+)
+
+// SettingsConfigAdmin is the typed side of SettingsAdmin. UpdateConfig runs mutate on the current configuration
+// under the same lock as every other settings write, then validates, saves and reloads it exactly like Apply
+// does for a YAML document. An error from mutate or from validation wraps ErrInvalidSettings.
+type SettingsConfigAdmin interface {
+	CurrentConfig() (cfg config.Config, restartRequired []string)
+	UpdateConfig(ctx context.Context, actor string, mutate func(*config.Config) error) (SettingsResult, error)
+}
+
+func (s *server) registerSettingsForm(api huma.API) {
+	huma.Register(api, huma.Operation{
+		OperationID: "get-settings-form", Method: http.MethodGet, Path: "/api/v1/settings/form", Tags: []string{"settings"},
+		Summary: "Read the configuration as a typed form", Description: "Admin only. The same configuration as GET /settings, " +
+			"as JSON sections, plus the defaults that apply to unset values.",
+		Errors: []int{http.StatusServiceUnavailable},
+	}, s.getSettingsForm)
+	huma.Register(api, huma.Operation{
+		OperationID: "put-settings-form", Method: http.MethodPut, Path: "/api/v1/settings/form", Tags: []string{"settings"},
+		Summary: "Replace configuration sections",
+		Description: "Admin only. Each section present in the body replaces that section; absent sections are untouched. " +
+			"The result is validated, saved and reloaded exactly like PUT /settings. An invalid value is a 422 whose errors " +
+			"locate the field, and nothing changes.",
+		Errors: []int{http.StatusUnprocessableEntity, http.StatusServiceUnavailable},
+	}, s.putSettingsForm)
+}
+
+func (s *server) formAdmin() (SettingsConfigAdmin, error) {
+	a, ok := s.Settings.(SettingsConfigAdmin)
+	if !ok {
+		return nil, errNotConfigured("settings")
+	}
+	return a, nil
+}
+
+func (s *server) getSettingsForm(context.Context, *struct{}) (*struct{ Body SettingsFormState }, error) {
+	a, err := s.formAdmin()
+	if err != nil {
+		return nil, err
+	}
+	cfg, pending := a.CurrentConfig()
+	v, err := formView(cfg)
+	if err != nil {
+		s.Logger.Error("read settings form", "err", err)
+		return nil, huma.Error500InternalServerError("could not read the configuration; see the log")
+	}
+	return &struct{ Body SettingsFormState }{SettingsFormState{Form: v, Defaults: formDefaults(), RestartRequired: nonNil(pending)}}, nil
+}
+
+func (s *server) putSettingsForm(ctx context.Context, in *struct{ Body SettingsFormPatch }) (*struct{ Body SettingsFormApplied }, error) {
+	a, err := s.formAdmin()
+	if err != nil {
+		return nil, err
+	}
+	res, err := a.UpdateConfig(ctx, actorOf(ctx), func(c *config.Config) error { return applyFormPatch(c, in.Body) })
+	if errors.Is(err, ErrInvalidSettings) {
+		return nil, settingsProblem(trimSentinel(err, ErrInvalidSettings), err)
+	}
+	if err != nil {
+		s.Logger.Error("apply settings form", "err", err)
+		return nil, huma.Error500InternalServerError("could not save the configuration; see the log")
+	}
+	cfg, pending := a.CurrentConfig()
+	v, err := formView(cfg)
+	if err != nil {
+		s.Logger.Warn("read back settings form", "err", err)
+	}
+	out := SettingsFormApplied{Form: v, Applied: nonNil(res.Applied), RestartRequired: nonNil(pending)}
+	return &struct{ Body SettingsFormApplied }{out}, nil
+}
+
+// validationPath finds the dotted config path a validation message starts with ("retention.hot must be ...",
+// "auth.oidc.issuer is required", "timezone: ...").
+var validationPath = regexp.MustCompile(`^([a-z_][a-z_0-9]*(?:\.[A-Za-z0-9_]+|\[\d+\])*)(:| )`)
+
+// settingsProblem is the 422 for a rejected form: the detail is the message and errors[0].location is
+// body.<path> when the message names a field, so the form can mark it.
+func settingsProblem(msg string, err error) error {
+	loc := ""
+	var fe *fieldError
+	if errors.As(err, &fe) {
+		loc, msg = fe.Loc, fe.Msg
+	} else if m := validationPath.FindStringSubmatch(msg); m != nil && (strings.Contains(m[1], ".") || m[2] == ":") {
+		loc = m[1]
+	}
+	if loc == "" {
+		return huma.Error422UnprocessableEntity(msg)
+	}
+	return huma.Error422UnprocessableEntity(msg, &huma.ErrorDetail{Message: msg, Location: "body." + loc})
+}

internal/api/settings_form_patch.go

@@ -0,0 +1,249 @@
+package api
+
+import (
+	"fmt"
+	"maps"
+	"strings"
+	"time"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/v10networks/nmon/internal/checks"
+	"github.com/v10networks/nmon/internal/config"
+	"github.com/v10networks/nmon/internal/notify"
+	"github.com/v10networks/nmon/internal/proxmox"
+)
+
+// fieldError is a form value that cannot even be turned into configuration (an unparsable duration); Loc is
+// the JSON path of the field.
+type fieldError struct{ Loc, Msg string }
+
+func (e *fieldError) Error() string { return e.Loc + ": " + e.Msg }
+
+// parseDur reads a form duration; empty is unset (zero).
+func parseDur(loc, s string) (time.Duration, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return 0, nil
+	}
+	d, err := config.ParseDuration(s)
+	if err != nil {
+		return 0, &fieldError{Loc: loc, Msg: fmt.Sprintf("%q is not a duration (examples: 30d, 5m, 1h30m)", s)}
+	}
+	return d, nil
+}
+
+// durs parses several durations in order, stopping at the first bad one.
+type durs struct{ err error }
+
+func (p *durs) get(loc, s string) time.Duration {
+	if p.err != nil {
+		return 0
+	}
+	d, err := parseDur(loc, s)
+	p.err = err
+	return d
+}
+
+// applyFormPatch replaces the sections present in p inside c. Validation of the result is the caller's job
+// and is the same as for a YAML document.
+func applyFormPatch(c *config.Config, p SettingsFormPatch) error {
+	steps := []func(*config.Config, SettingsFormPatch) error{
+		patchRetention, patchGeneral, patchSites, patchAlerting, patchReceivers, patchAuth, patchBackupDisk,
+		patchL2OUI, patchProxmox, patchChecks,
+	}
+	for _, step := range steps {
+		if err := step(c, p); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func patchRetention(c *config.Config, p SettingsFormPatch) error {
+	if p.Retention == nil {
+		return nil
+	}
+	var d durs
+	r := config.Retention{Hot: config.Duration(d.get("retention.hot", p.Retention.Hot)),
+		FiveMinute: config.Duration(d.get("retention.fiveminute", p.Retention.FiveMinute)),
+		Long:       config.Duration(d.get("retention.long", p.Retention.Long))}
+	c.Retention = r
+	return d.err
+}
+
+func patchGeneral(c *config.Config, p SettingsFormPatch) error {
+	if p.Timezone != nil {
+		c.Timezone = strings.TrimSpace(*p.Timezone)
+	}
+	var d durs
+	if p.RetiredPurgeAfter != nil {
+		c.RetiredPurgeAfter = config.Duration(d.get("retired_purge_after", *p.RetiredPurgeAfter))
+	}
+	if p.PollerInterval != nil {
+		c.Poller.Interval = config.Duration(d.get("poller.interval", *p.PollerInterval))
+	}
+	return d.err
+}
+
+func patchSites(c *config.Config, p SettingsFormPatch) error {
+	if p.Sites == nil {
+		return nil
+	}
+	sites := make(map[string]config.Site, len(*p.Sites))
+	for i, s := range *p.Sites {
+		name := strings.TrimSpace(s.Name)
+		if _, dup := sites[name]; name == "" || dup {
+			return &fieldError{Loc: fmt.Sprintf("sites[%d].name", i), Msg: "site names must be non-empty and unique"}
+		}
+		sites[name] = config.Site{Timezone: strings.TrimSpace(s.Timezone)}
+	}
+	if len(sites) == 0 {
+		sites = nil
+	}
+	c.Sites = sites
+	return nil
+}
+
+func patchAlerting(c *config.Config, p SettingsFormPatch) error {
+	if p.Alerting == nil {
+		return nil
+	}
+	var d durs
+	c.Alerting.EvaluationInterval = config.Duration(d.get("alerting.evaluation_interval", p.Alerting.EvaluationInterval))
+	c.Alerting.OutboundAllow = cleanList(p.Alerting.OutboundAllow)
+	return d.err
+}
+
+// cleanList trims entries and drops blanks; an empty list is nil so it is omitted from config.yaml.
+func cleanList(in []string) []string {
+	var out []string
+	for _, s := range in {
+		if s = strings.TrimSpace(s); s != "" {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+// storedReceivers are the receivers now in the configuration, by name, so that write-only credentials the form
+// leaves blank can be carried over.
+func storedReceivers(a config.Alerting) map[string]notify.Receiver {
+	out := make(map[string]notify.Receiver, len(a.Receivers))
+	for i := range a.Receivers {
+		var r notify.Receiver
+		if err := a.Receivers[i].Decode(&r); err == nil {
+			out[r.Name] = r
+		}
+	}
+	return out
+}
+
+// receiverFrom builds a receiver from its form, keeping the stored credentials the form left blank.
+func receiverFrom(f ReceiverForm, old notify.Receiver) notify.Receiver {
+	headers := maps.Clone(f.Headers)
+	for k, v := range headers {
+		if stored, ok := old.Headers[k]; ok && v == "" {
+			headers[k] = stored
+		}
+	}
+	password := f.SMTP.Password
+	if password == "" && !f.SMTP.ClearPassword {
+		password = old.SMTP.Password
+	}
+	return notify.Receiver{Name: strings.TrimSpace(f.Name), Kind: f.Kind, URL: strings.TrimSpace(f.URL),
+		Headers: headers, Template: f.Template,
+		SMTP: notify.SMTP{Host: strings.TrimSpace(f.SMTP.Host), Port: f.SMTP.Port, From: strings.TrimSpace(f.SMTP.From),
+			To: cleanList(f.SMTP.To), Username: f.SMTP.Username, Password: password, StartTLS: f.SMTP.StartTLS}}
+}
+
+func patchReceivers(c *config.Config, p SettingsFormPatch) error {
+	if p.Receivers == nil {
+		return nil
+	}
+	stored := storedReceivers(c.Alerting)
+	var nodes []yaml.Node
+	for i, f := range *p.Receivers {
+		var n yaml.Node
+		if err := n.Encode(receiverFrom(f, stored[strings.TrimSpace(f.Name)])); err != nil {
+			return &fieldError{Loc: fmt.Sprintf("receivers[%d]", i), Msg: err.Error()}
+		}
+		nodes = append(nodes, n)
+	}
+	c.Alerting.Receivers = nodes
+	return nil
+}
+
+func patchAuth(c *config.Config, p SettingsFormPatch) error {
+	if p.Auth == nil {
+		return nil
+	}
+	f := p.Auth
+	var d durs
+	a := config.Auth{BreakGlassFile: strings.TrimSpace(f.BreakGlassFile), BreakGlassUser: strings.TrimSpace(f.BreakGlassUser),
+		TrustedProxies: cleanList(f.TrustedProxies),
+		Session: config.Session{Idle: config.Duration(d.get("auth.session.idle", f.Session.Idle)),
+			Absolute: config.Duration(d.get("auth.session.absolute", f.Session.Absolute))}}
+	if o := f.OIDC; o != nil {
+		a.OIDC = &config.OIDC{Issuer: strings.TrimSpace(o.Issuer), ClientID: strings.TrimSpace(o.ClientID),
+			ClientSecretName: strings.TrimSpace(o.ClientSecretName), RedirectURL: strings.TrimSpace(o.RedirectURL),
+			GroupsClaim: strings.TrimSpace(o.GroupsClaim), AdminGroup: strings.TrimSpace(o.AdminGroup)}
+	}
+	c.Auth = a
+	return d.err
+}
+
+func patchBackupDisk(c *config.Config, p SettingsFormPatch) error {
+	if b := p.Backup; b != nil {
+		c.Backup = config.Backup{At: strings.TrimSpace(b.At), Dir: strings.TrimSpace(b.Dir), Keep: b.Keep}
+	}
+	if d := p.Disk; d != nil {
+		c.Disk = config.Disk{AlertUsed: d.AlertUsed, AdmitUsed: d.AdmitUsed, ReserveFree: d.ReserveFree}
+	}
+	return nil
+}
+
+func patchL2OUI(c *config.Config, p SettingsFormPatch) error {
+	var d durs
+	if l := p.L2; l != nil {
+		c.L2 = config.L2{Interval: config.Duration(d.get("l2.interval", l.Interval)), Keep: config.Duration(d.get("l2.keep", l.Keep)),
+			EdgeMaxMACs: l.EdgeMaxMACs, Concurrency: l.Concurrency,
+			DeviceTimeout: config.Duration(d.get("l2.device_timeout", l.DeviceTimeout))}
+	}
+	if o := p.OUI; o != nil {
+		oui := config.OUI{Refresh: config.Duration(d.get("oui.refresh", o.Refresh))}
+		if !o.Fetch { // fetch defaults to on, so only "off" is written
+			off := false
+			oui.Fetch = &off
+		}
+		c.OUI = oui
+	}
+	return d.err
+}
+
+func patchProxmox(c *config.Config, p SettingsFormPatch) error {
+	f := p.Proxmox
+	if f == nil {
+		return nil
+	}
+	var d durs
+	pc := proxmox.Config{Interval: d.get("proxmox.interval", f.Interval), Timeout: d.get("proxmox.timeout", f.Timeout),
+		RetireGrace: d.get("proxmox.retire_grace", f.RetireGrace), Workers: f.Workers}
+	for _, cl := range f.Clusters {
+		pc.Clusters = append(pc.Clusters, proxmox.ClusterConfig{Name: strings.TrimSpace(cl.Name), URL: strings.TrimSpace(cl.URL),
+			TokenID: strings.TrimSpace(cl.TokenID), SecretName: strings.TrimSpace(cl.SecretName), CAFile: strings.TrimSpace(cl.CAFile),
+			InsecureSkipVerify: cl.InsecureSkipVerify})
+	}
+	c.Proxmox = pc
+	return d.err
+}
+
+func patchChecks(c *config.Config, p SettingsFormPatch) error {
+	f := p.Checks
+	if f == nil {
+		return nil
+	}
+	var d durs
+	c.Checks = checks.Config{Workers: f.Workers, Timeout: d.get("checks.timeout", f.Timeout), Tick: d.get("checks.tick", f.Tick)}
+	return d.err
+}

internal/api/settings_form_test.go

@@ -0,0 +1,135 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/auth"
+	"github.com/v10networks/nmon/internal/auth/authtest"
+	"github.com/v10networks/nmon/internal/config"
+	"github.com/v10networks/nmon/internal/notify"
+)
+
+// formSettings keeps a config in memory and validates it with config.Validate, as the app does.
+type formSettings struct {
+	fakeSettings
+	cfg config.Config
+	mu2 sync.Mutex
+}
+
+func (f *formSettings) CurrentConfig() (config.Config, []string) {
+	f.mu2.Lock()
+	defer f.mu2.Unlock()
+	return f.cfg, nil
+}
+
+func (f *formSettings) UpdateConfig(_ context.Context, _ string, mutate func(*config.Config) error) (api.SettingsResult, error) {
+	f.mu2.Lock()
+	defer f.mu2.Unlock()
+	c := f.cfg
+	if err := mutate(&c); err != nil {
+		return api.SettingsResult{}, fmt.Errorf("%w: %w", api.ErrInvalidSettings, err)
+	}
+	if err := c.Validate(); err != nil {
+		return api.SettingsResult{}, fmt.Errorf("%w: %w", api.ErrInvalidSettings, err)
+	}
+	f.cfg = c
+	return api.SettingsResult{Applied: []string{"backup"}}, nil
+}
+
+func TestSettingsFormReadsTypedSectionsAndPatchesOnlyThePresentOnes(t *testing.T) {
+	fs := &formSettings{cfg: config.Default()}
+	e := newAuthedEnv(t, func(d *api.Deps) { d.Settings = fs })
+	admin, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
+	viewer, _ := authtest.Mint(t, e.auth, auth.RoleViewer)
+
+	if code := e.call(http.MethodGet, "/api/v1/settings/form", "", viewer).Code; code != http.StatusForbidden {
+		t.Fatalf("a viewer reading the form: %d, want 403", code)
+	}
+	var st api.SettingsFormState
+	rec := e.call(http.MethodGet, "/api/v1/settings/form", "", admin)
+	if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil || st.Form.Retention.Hot != "30d" || st.Defaults.L2.Interval != "15m" {
+		t.Fatalf("form: %d %s", rec.Code, rec.Body)
+	}
+
+	rec = e.call(http.MethodPut, "/api/v1/settings/form", `{"backup":{"at":"03:00","dir":"","keep":5},"retention":{"hot":"45d","fiveminute":"120d","long":"2y"}}`, admin)
+	var out api.SettingsFormApplied
+	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil || rec.Code != http.StatusOK || out.Form.Backup.At != "03:00" || out.Form.Retention.Hot != "45d" {
+		t.Fatalf("patch: %d %s", rec.Code, rec.Body)
+	}
+	if fs.cfg.Timezone != "UTC" || fs.cfg.Backup.Keep != 5 {
+		t.Fatalf("config = %+v", fs.cfg)
+	}
+}
+
+func TestSettingsFormRejectionsLocateTheField(t *testing.T) {
+	fs := &formSettings{cfg: config.Default()}
+	e := newAuthedEnv(t, func(d *api.Deps) { d.Settings = fs })
+	admin, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
+	for body, loc := range map[string]string{
+		`{"retention":{"hot":"12h","fiveminute":"120d","long":"2y"}}`:                              "body.retention.hot",
+		`{"retention":{"hot":"soon","fiveminute":"120d","long":"2y"}}`:                             "body.retention.hot",
+		`{"timezone":"Mars/Base"}`:                                                                 "body.timezone",
+		`{"backup":{"at":"25:00","dir":"","keep":0}}`:                                              "body.backup.at",
+		`{"sites":[{"name":"a","timezone":""},{"name":"a","timezone":""}]}`:                        "body.sites[1].name",
+		`{"disk":{"alert_used":0.95,"admit_used":0.9,"reserve_free":0}}`:                           "body.disk",
+		`{"l2":{"interval":"1m","keep":"","edge_max_macs":0,"concurrency":0,"device_timeout":""}}`: "body.l2.interval",
+	} {
+		rec := e.call(http.MethodPut, "/api/v1/settings/form", body, admin)
+		if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), `"location":"`+loc) {
+			t.Errorf("%s: %d %s (want location %s)", body, rec.Code, rec.Body, loc)
+		}
+	}
+	if fs.cfg.Retention.Hot.D() != config.Default().Retention.Hot.D() {
+		t.Fatalf("a rejected form changed the configuration: %+v", fs.cfg)
+	}
+}
+
+func TestSettingsFormReceiverCredentialsAreWriteOnly(t *testing.T) {
+	fs := &formSettings{cfg: config.Default()}
+	e := newAuthedEnv(t, func(d *api.Deps) { d.Settings = fs })
+	admin, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
+	put := func(body string) string {
+		rec := e.call(http.MethodPut, "/api/v1/settings/form", body, admin)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("PUT %s: %d %s", body, rec.Code, rec.Body)
+		}
+		return rec.Body.String()
+	}
+	const smtp = `{"name":"mail","kind":"smtp","url":"","headers":{},"template":"","smtp":{"host":"smtp.example.net","port":0,"from":"a@b.c","to":["x@y.z"],` +
+		`"username":"u","password":%q,"clear_password":%t,"starttls":true}}`
+	const hook = `{"name":"hook","kind":"webhook","url":"https://h.example.net/x","headers":{"Authorization":%q},"template":"","smtp":{"host":"","port":0,"from":"",` +
+		`"to":[],"username":"","password":"","clear_password":false,"starttls":false}}`
+	out := put(`{"receivers":[` + fmt.Sprintf(smtp, "hunter2", false) + `,` + fmt.Sprintf(hook, "Bearer abc") + `]}`)
+	if strings.Contains(out, "hunter2") || strings.Contains(out, "Bearer abc") || !strings.Contains(out, `"login_stored":true`) {
+		t.Fatalf("the response leaked or hid a credential wrongly: %s", out)
+	}
+	put(`{"receivers":[` + fmt.Sprintf(smtp, "", false) + `,` + fmt.Sprintf(hook, "") + `]}`)
+	rs := alertingReceivers(t, fs)
+	if rs[0].SMTP.Password != "hunter2" || rs[1].Headers["Authorization"] != "Bearer abc" {
+		t.Fatalf("blank credentials must keep the stored ones: %+v", rs)
+	}
+	put(`{"receivers":[` + fmt.Sprintf(smtp, "", true) + `]}`)
+	if rs = alertingReceivers(t, fs); rs[0].SMTP.Password != "" {
+		t.Fatalf("clear_password kept the password: %+v", rs)
+	}
+}
+
+func alertingReceivers(t *testing.T, fs *formSettings) []notify.Receiver {
+	t.Helper()
+	var out []notify.Receiver
+	for i := range fs.cfg.Alerting.Receivers {
+		var r notify.Receiver
+		if err := fs.cfg.Alerting.Receivers[i].Decode(&r); err != nil {
+			t.Fatal(err)
+		}
+		out = append(out, r)
+	}
+	return out
+}

internal/api/settings_form_types.go

@@ -0,0 +1,201 @@
+package api
+
+// The typed settings form. It carries the same sections as config.yaml, as JSON with plain strings for
+// durations ("30d", "5m", "1h30m": config.ParseDuration syntax, empty = unset, so the default applies).
+// Every section type is replaced as a whole when it is written. Secrets are never part of it: the forms name
+// a stored secret and the value is set through PUT /api/v1/secrets/{name}.
+
+// RetentionForm is the retention section.
+type RetentionForm struct {
+	Hot        string `json:"hot" doc:"Raw samples in the hot store. At least 1d. Default 30d."`
+	FiveMinute string `json:"fiveminute" doc:"Five-minute rates in the long store. At least retention.hot. Default 120d."`
+	Long       string `json:"long" doc:"Hourly rates in the long store. At most 10y. Default 2y."`
+}
+
+// SiteForm is one entry of the sites section.
+type SiteForm struct {
+	Name     string `json:"name" doc:"Device site name (devices.site)."`
+	Timezone string `json:"timezone" doc:"IANA zone whose calendar months define this site's p95. Empty: the top-level timezone."`
+}
+
+// AlertingForm is the part of the alerting section that is plain data. Alert rules are managed under Alerts
+// and receivers have their own list.
+type AlertingForm struct {
+	OutboundAllow      []string `json:"outbound_allow" doc:"CIDRs, addresses and host names webhooks and SMTP may reach beyond public ones."`
+	EvaluationInterval string   `json:"evaluation_interval" doc:"How often metric rules run. At least 1s. Default 30s."`
+}
+
+// SMTPForm is the smtp part of a receiver, as written.
+type SMTPForm struct {
+	Host          string   `json:"host"`
+	Port          int      `json:"port" doc:"0: 587 with StartTLS, otherwise 25."`
+	From          string   `json:"from"`
+	To            []string `json:"to"`
+	Username      string   `json:"username"`
+	Password      string   `json:"password" doc:"Write-only. Empty keeps the stored one of the receiver with the same name."`
+	ClearPassword bool     `json:"clear_password" doc:"Remove the stored SMTP password."`
+	StartTLS      bool     `json:"starttls"`
+}
+
+// ReceiverForm is one notification receiver, as written. Credentials (header values, the SMTP password) are
+// write-only: an empty value keeps what the receiver of the same name already has.
+type ReceiverForm struct {
+	Name     string            `json:"name"`
+	Kind     string            `json:"kind" enum:"webhook,smtp"`
+	URL      string            `json:"url" doc:"Webhook endpoint. May embed a token."`
+	Headers  map[string]string `json:"headers" doc:"Extra webhook headers. An empty value keeps the stored value of the same-named header."`
+	Template string            `json:"template" doc:"Go text/template; empty selects the default for the kind."`
+	SMTP     SMTPForm          `json:"smtp"`
+}
+
+// SMTPView is the smtp part of a receiver, as read: the password is never returned.
+type SMTPView struct {
+	Host        string   `json:"host"`
+	Port        int      `json:"port"`
+	From        string   `json:"from"`
+	To          []string `json:"to"`
+	Username    string   `json:"username"`
+	LoginStored bool     `json:"login_stored" doc:"An SMTP password is stored. It cannot be read back."`
+	StartTLS    bool     `json:"starttls"`
+}
+
+// ReceiverView is one notification receiver, as read. Header values are blank: they may be credentials.
+type ReceiverView struct {
+	Name     string            `json:"name"`
+	Kind     string            `json:"kind" enum:"webhook,smtp"`
+	URL      string            `json:"url"`
+	Headers  map[string]string `json:"headers" doc:"Header names with blank values; the values cannot be read back."`
+	Template string            `json:"template"`
+	SMTP     SMTPView          `json:"smtp"`
+}
+
+// OIDCForm is the OpenID Connect provider.
+type OIDCForm struct {
+	Issuer           string `json:"issuer"`
+	ClientID         string `json:"client_id"`
+	ClientSecretName string `json:"client_secret_name" doc:"Name of the stored secret holding the client secret."`
+	RedirectURL      string `json:"redirect_url" doc:"nmon's callback, .../auth/callback."`
+	GroupsClaim      string `json:"groups_claim"`
+	AdminGroup       string `json:"admin_group" doc:"Members are admins; empty means nobody is."`
+}
+
+// SessionForm holds session lifetimes.
+type SessionForm struct {
+	Idle     string `json:"idle" doc:"Default 1h."`
+	Absolute string `json:"absolute" doc:"Default 12h."`
+}
+
+// AuthForm is the auth section.
+type AuthForm struct {
+	OIDC           *OIDCForm   `json:"oidc,omitempty" doc:"Absent: only break-glass and API tokens can sign in."`
+	BreakGlassFile string      `json:"break_glass_file"`
+	BreakGlassUser string      `json:"break_glass_user" doc:"Used when the file holds a bare hash. Default breakglass."`
+	TrustedProxies []string    `json:"trusted_proxies" doc:"Addresses or CIDRs whose X-Forwarded-* headers are honoured."`
+	Session        SessionForm `json:"session"`
+}
+
+// BackupForm is the backup section.
+type BackupForm struct {
+	At   string `json:"at" doc:"HH:MM local time of the nightly backup; empty disables the schedule."`
+	Dir  string `json:"dir" doc:"Default <data-dir>/backups."`
+	Keep int    `json:"keep" doc:"Scheduled backups to retain. 0: default 7."`
+}
+
+// DiskForm is the disk section, as fractions of the filesystem.
+type DiskForm struct {
+	AlertUsed   float64 `json:"alert_used" doc:"Raises the disk alert. 0: default 0.80."`
+	AdmitUsed   float64 `json:"admit_used" doc:"Stops new series in the hot store. 0: default 0.90."`
+	ReserveFree float64 `json:"reserve_free" doc:"Free fraction below which ingest is throttled. 0: default 0.05."`
+}
+
+// L2Form is the l2 section.
+type L2Form struct {
+	Interval      string `json:"interval" doc:"At least 5m. Default 15m."`
+	Keep          string `json:"keep" doc:"At least 1d. Default 90d."`
+	EdgeMaxMACs   int    `json:"edge_max_macs" doc:"Default 8."`
+	Concurrency   int    `json:"concurrency" doc:"1 to 64. Default 4."`
+	DeviceTimeout string `json:"device_timeout" doc:"At least 10s. Default 2m."`
+}
+
+// OUIForm is the oui section.
+type OUIForm struct {
+	Fetch   bool   `json:"fetch" doc:"Download the IEEE vendor registries. Default true."`
+	Refresh string `json:"refresh" doc:"At least 1h. Default 7d."`
+}
+
+// ProxmoxClusterForm is one Proxmox cluster.
+type ProxmoxClusterForm struct {
+	Name               string `json:"name"`
+	URL                string `json:"url"`
+	TokenID            string `json:"token_id" doc:"user@realm!tokenid."`
+	SecretName         string `json:"secret_name" doc:"Name of the stored secret holding the token."`
+	CAFile             string `json:"ca_file"`
+	InsecureSkipVerify bool   `json:"insecure_skip_verify"`
+}
+
+// ProxmoxForm is the proxmox section.
+type ProxmoxForm struct {
+	Clusters    []ProxmoxClusterForm `json:"clusters"`
+	Interval    string               `json:"interval" doc:"Default 5m."`
+	Timeout     string               `json:"timeout" doc:"Default 60s."`
+	RetireGrace string               `json:"retire_grace" doc:"Default 24h. Negative never retires."`
+	Workers     int                  `json:"workers" doc:"Default 4."`
+}
+
+// ChecksForm is the checks section (the checks themselves live in the inventory).
+type ChecksForm struct {
+	Workers int    `json:"workers" doc:"Default 32."`
+	Timeout string `json:"timeout" doc:"Default 10s."`
+	Tick    string `json:"tick" doc:"Default 5s."`
+}
+
+// SettingsFormView is every form section as the server has it.
+type SettingsFormView struct {
+	Retention         RetentionForm  `json:"retention"`
+	Timezone          string         `json:"timezone" doc:"IANA name; the site default for p95 months."`
+	RetiredPurgeAfter string         `json:"retired_purge_after" doc:"Default 1y."`
+	PollerInterval    string         `json:"poller_interval" doc:"Default SNMP poll interval, 30s to 1h. Default 5m. A device's own wins."`
+	Sites             []SiteForm     `json:"sites"`
+	Alerting          AlertingForm   `json:"alerting"`
+	Receivers         []ReceiverView `json:"receivers"`
+	Auth              AuthForm       `json:"auth"`
+	Backup            BackupForm     `json:"backup"`
+	Disk              DiskForm       `json:"disk"`
+	L2                L2Form         `json:"l2"`
+	OUI               OUIForm        `json:"oui"`
+	Proxmox           ProxmoxForm    `json:"proxmox"`
+	Checks            ChecksForm     `json:"checks"`
+}
+
+// SettingsFormState is GET /settings/form.
+type SettingsFormState struct {
+	Form            SettingsFormView `json:"form" doc:"The saved configuration. A zero or empty value means the default applies."`
+	Defaults        SettingsFormView `json:"defaults" doc:"What an unset value means, for placeholders."`
+	RestartRequired []string         `json:"restart_required" doc:"Sections whose saved value differs from what is running."`
+}
+
+// SettingsFormPatch is the body of PUT /settings/form: each section present replaces that section, the others
+// are left alone.
+type SettingsFormPatch struct {
+	Retention         *RetentionForm  `json:"retention,omitempty"`
+	Timezone          *string         `json:"timezone,omitempty"`
+	RetiredPurgeAfter *string         `json:"retired_purge_after,omitempty"`
+	PollerInterval    *string         `json:"poller_interval,omitempty"`
+	Sites             *[]SiteForm     `json:"sites,omitempty"`
+	Alerting          *AlertingForm   `json:"alerting,omitempty"`
+	Receivers         *[]ReceiverForm `json:"receivers,omitempty"`
+	Auth              *AuthForm       `json:"auth,omitempty"`
+	Backup            *BackupForm     `json:"backup,omitempty"`
+	Disk              *DiskForm       `json:"disk,omitempty"`
+	L2                *L2Form         `json:"l2,omitempty"`
+	OUI               *OUIForm        `json:"oui,omitempty"`
+	Proxmox           *ProxmoxForm    `json:"proxmox,omitempty"`
+	Checks            *ChecksForm     `json:"checks,omitempty"`
+}
+
+// SettingsFormApplied is the outcome of PUT /settings/form.
+type SettingsFormApplied struct {
+	Form            SettingsFormView `json:"form" doc:"The configuration as saved."`
+	Applied         []string         `json:"applied" doc:"Sections reloaded in place."`
+	RestartRequired []string         `json:"restart_required" doc:"Sections saved but running with the old value until a restart."`
+}

internal/api/settings_form_view.go

@@ -0,0 +1,145 @@
+package api
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/v10networks/nmon/internal/auth"
+	"github.com/v10networks/nmon/internal/config"
+	"github.com/v10networks/nmon/internal/diskguard"
+	"github.com/v10networks/nmon/internal/notify"
+	"github.com/v10networks/nmon/internal/proxmox"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// durString renders a duration for the form; zero is "unset" and shows as empty.
+func durString(d time.Duration) string {
+	if d == 0 {
+		return ""
+	}
+	s := config.FormatDuration(d)
+	// Go prints 15m as "15m0s" and 1h as "1h0m0s"; the form shows the short spelling.
+	if strings.HasSuffix(s, "m0s") {
+		s = strings.TrimSuffix(s, "0s")
+	}
+	if strings.HasSuffix(s, "h0m") {
+		s = strings.TrimSuffix(s, "0m")
+	}
+	return s
+}
+
+// formView converts the configuration into the typed form.
+func formView(c config.Config) (SettingsFormView, error) {
+	recv, err := receiverViews(c.Alerting)
+	if err != nil {
+		return SettingsFormView{}, err
+	}
+	v := SettingsFormView{
+		Retention: RetentionForm{Hot: durString(c.Retention.Hot.D()), FiveMinute: durString(c.Retention.FiveMinute.D()),
+			Long: durString(c.Retention.Long.D())},
+		Timezone:          c.Timezone,
+		RetiredPurgeAfter: durString(c.RetiredPurgeAfter.D()),
+		PollerInterval:    durString(c.Poller.Interval.D()),
+		Sites:             siteForms(c.Sites),
+		Alerting: AlertingForm{OutboundAllow: nonNil(c.Alerting.OutboundAllow),
+			EvaluationInterval: durString(c.Alerting.EvaluationInterval.D())},
+		Receivers: recv,
+		Auth:      authForm(c.Auth),
+		Backup:    BackupForm{At: c.Backup.At, Dir: c.Backup.Dir, Keep: c.Backup.Keep},
+		Disk:      DiskForm{AlertUsed: c.Disk.AlertUsed, AdmitUsed: c.Disk.AdmitUsed, ReserveFree: c.Disk.ReserveFree},
+		L2: L2Form{Interval: durString(c.L2.Interval.D()), Keep: durString(c.L2.Keep.D()), EdgeMaxMACs: c.L2.EdgeMaxMACs,
+			Concurrency: c.L2.Concurrency, DeviceTimeout: durString(c.L2.DeviceTimeout.D())},
+		OUI:     OUIForm{Fetch: c.OUI.FetchEnabled(), Refresh: durString(c.OUI.Refresh.D())},
+		Proxmox: proxmoxForm(c.Proxmox),
+		Checks:  ChecksForm{Workers: c.Checks.Workers, Timeout: durString(c.Checks.Timeout), Tick: durString(c.Checks.Tick)},
+	}
+	return v, nil
+}
+
+func siteForms(sites map[string]config.Site) []SiteForm {
+	out := make([]SiteForm, 0, len(sites))
+	for name, s := range sites {
+		out = append(out, SiteForm{Name: name, Timezone: s.Timezone})
+	}
+	slices.SortFunc(out, func(a, b SiteForm) int { return compareStrings(a.Name, b.Name) })
+	return out
+}
+
+func compareStrings(a, b string) int {
+	switch {
+	case a < b:
+		return -1
+	case a > b:
+		return 1
+	}
+	return 0
+}
+
+func receiverViews(a config.Alerting) ([]ReceiverView, error) {
+	out := make([]ReceiverView, 0, len(a.Receivers))
+	for i := range a.Receivers {
+		var r notify.Receiver
+		if err := a.Receivers[i].Decode(&r); err != nil {
+			return nil, fmt.Errorf("alerting.receivers[%d]: %w", i, err)
+		}
+		headers := make(map[string]string, len(r.Headers)) // names only: the values may be credentials
+		for k := range r.Headers {
+			headers[k] = ""
+		}
+		out = append(out, ReceiverView{Name: r.Name, Kind: r.Kind, URL: r.URL, Headers: headers, Template: r.Template,
+			SMTP: SMTPView{Host: r.SMTP.Host, Port: r.SMTP.Port, From: r.SMTP.From, To: nonNil(r.SMTP.To),
+				Username: r.SMTP.Username, LoginStored: r.SMTP.Password != "", StartTLS: r.SMTP.StartTLS}})
+	}
+	return out, nil
+}
+
+func authForm(a config.Auth) AuthForm {
+	f := AuthForm{BreakGlassFile: a.BreakGlassFile, BreakGlassUser: a.BreakGlassUser, TrustedProxies: nonNil(a.TrustedProxies),
+		Session: SessionForm{Idle: durString(a.Session.Idle.D()), Absolute: durString(a.Session.Absolute.D())}}
+	if o := a.OIDC; o != nil {
+		f.OIDC = &OIDCForm{Issuer: o.Issuer, ClientID: o.ClientID, ClientSecretName: o.ClientSecretName, RedirectURL: o.RedirectURL,
+			GroupsClaim: o.GroupsClaim, AdminGroup: o.AdminGroup}
+	}
+	return f
+}
+
+func proxmoxForm(p proxmox.Config) ProxmoxForm {
+	clusters := make([]ProxmoxClusterForm, 0, len(p.Clusters))
+	for _, c := range p.Clusters {
+		clusters = append(clusters, ProxmoxClusterForm{Name: c.Name, URL: c.URL, TokenID: c.TokenID, SecretName: c.SecretName,
+			CAFile: c.CAFile, InsecureSkipVerify: c.InsecureSkipVerify})
+	}
+	return ProxmoxForm{Clusters: clusters, Interval: durString(p.Interval), Timeout: durString(p.Timeout),
+		RetireGrace: durString(p.RetireGrace), Workers: p.Workers}
+}
+
+// formDefaults is what each unset value means; the UI shows it as a placeholder.
+func formDefaults() SettingsFormView {
+	d := config.Default()
+	return SettingsFormView{
+		Retention: RetentionForm{Hot: durString(d.Retention.Hot.D()), FiveMinute: durString(d.Retention.FiveMinute.D()),
+			Long: durString(d.Retention.Long.D())},
+		Timezone:          d.Timezone,
+		RetiredPurgeAfter: durString(config.DefaultRetiredPurgeAfter),
+		PollerInterval:    durString(store.DefaultPollInterval),
+		Sites:             []SiteForm{},
+		Alerting:          AlertingForm{OutboundAllow: []string{}, EvaluationInterval: "30s"},
+		Receivers:         []ReceiverView{},
+		Auth: AuthForm{BreakGlassUser: "breakglass", TrustedProxies: []string{}, Session: SessionForm{
+			Idle: durString(auth.DefaultIdleTimeout), Absolute: durString(auth.DefaultAbsoluteTimeout)}},
+		Backup: BackupForm{Keep: config.DefaultBackupKeep},
+		Disk: DiskForm{AlertUsed: diskguard.DefaultAlertUsed, AdmitUsed: diskguard.DefaultAdmitUsed,
+			ReserveFree: diskguard.DefaultReserveFree},
+		L2: L2Form{Interval: durString(config.DefaultL2Interval), Keep: durString(config.DefaultL2Keep),
+			EdgeMaxMACs: config.DefaultL2EdgeMaxMACs, Concurrency: config.DefaultL2Concurrency,
+			DeviceTimeout: durString(config.DefaultL2DeviceTimeout)},
+		OUI: OUIForm{Fetch: true, Refresh: durString(config.DefaultOUIRefresh)},
+		Proxmox: ProxmoxForm{Clusters: []ProxmoxClusterForm{}, Interval: durString(proxmox.DefaultInterval),
+			Timeout: durString(proxmox.DefaultTimeout), RetireGrace: durString(proxmox.DefaultRetireGrace),
+			Workers: proxmox.DefaultWorkers},
+		// The checks runner's defaults are literals in internal/checks (Workers 32, Timeout 10s, Tick 5s).
+		Checks: ChecksForm{Workers: 32, Timeout: "10s", Tick: "5s"},
+	}
+}

internal/api/setup.go

@@ -0,0 +1,153 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/danielgtaylor/huma/v2"
+
+	"github.com/v10networks/nmon/internal/auth"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// Setup errors the app returns; the handlers map them to 409.
+var (
+	// ErrSetupClosed means install mode is off: setup was completed, or a sign-in was configured from the start.
+	ErrSetupClosed = errors.New("setup is closed")
+	// ErrSetupNoSignIn means setup cannot finish because no sign-in method is configured yet.
+	ErrSetupNoSignIn = errors.New("no sign-in method is configured yet")
+)
+
+// OIDCSecretName is where the wizard stores the OIDC client secret.
+const OIDCSecretName = "oidc-client"
+
+// SetupAdmin is the part of the running app the wizard drives. Install mode is a property of the process, so the
+// API asks the app rather than keeping its own copy.
+type SetupAdmin interface {
+	// InstallMode reports whether install mode is on right now.
+	InstallMode() bool
+	// SetupCompleted reports whether setup was ever completed (the stored one-way flag).
+	SetupCompleted(ctx context.Context) (bool, error)
+	// SignIn writes the break-glass file, stores the OIDC secret and sets auth.* in config.yaml. A bad value wraps
+	// ErrInvalidSettings; install mode being off is ErrSetupClosed.
+	SignIn(ctx context.Context, actor string, in SetupSignInBody) (SetupSignIn, error)
+	// Complete re-checks the sign-in, marks setup completed for good and swaps the live authentication in. It
+	// returns ErrSetupClosed or ErrSetupNoSignIn.
+	Complete(ctx context.Context, actor string) (SetupComplete, error)
+}
+
+func (s *server) registerSetup(api huma.API) {
+	tags := []string{"setup"}
+	huma.Register(api, huma.Operation{
+		OperationID: "get-setup", Method: http.MethodGet, Path: "/api/v1/setup", Tags: tags,
+		Summary: "First-install state",
+		Description: "Public. Says whether install mode is on (the UI then sends every page to /setup) and whether setup was " +
+			"completed. Progress counts are filled in only while install mode is on.",
+	}, s.getSetup)
+	huma.Register(api, huma.Operation{
+		OperationID: "setup-sign-in", Method: http.MethodPost, Path: "/api/v1/setup/signin", Tags: tags,
+		Summary: "Set up sign-in (install mode only)",
+		Description: "Creates the break-glass admin (bcrypt, written to <data-dir>/breakglass with mode 0600) and optionally " +
+			"OIDC. 409 once setup is completed.",
+		Errors: []int{http.StatusConflict, http.StatusUnprocessableEntity, http.StatusServiceUnavailable},
+	}, s.setupSignIn)
+	huma.Register(api, huma.Operation{
+		OperationID: "setup-recommended-policies", Method: http.MethodPost, Path: "/api/v1/setup/recommended-policies", Tags: tags,
+		Summary: "Create the recommended alert policies (install mode only)",
+		Description: "device_down, device_warning, interface_errors, cpu_high, memory_high and disk_full, each for everything. " +
+			"Idempotent: a policy that exists by name, or any policy already bound to the rule, is left alone.",
+		Errors: []int{http.StatusConflict, http.StatusServiceUnavailable},
+	}, s.setupPolicies)
+	huma.Register(api, huma.Operation{
+		OperationID: "setup-complete", Method: http.MethodPost, Path: "/api/v1/setup/complete", Tags: tags,
+		Summary: "Finish setup (install mode only)",
+		Description: "409 unless a sign-in method is configured. Ends install mode permanently and switches authentication on " +
+			"without a restart; the browser then signs in at the returned URL.",
+		Errors: []int{http.StatusConflict, http.StatusServiceUnavailable},
+	}, s.setupComplete)
+}
+
+func (s *server) setupAdmin() (SetupAdmin, error) {
+	if s.Setup == nil {
+		return nil, errNotConfigured("setup")
+	}
+	return s.Setup, nil
+}
+
+func (s *server) getSetup(ctx context.Context, _ *struct{}) (*struct{ Body SetupState }, error) {
+	a, err := s.setupAdmin()
+	if err != nil {
+		return nil, err
+	}
+	done, err := a.SetupCompleted(ctx)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	out := SetupState{InstallMode: a.InstallMode(), Completed: done, Defaults: setupDefaults()}
+	if out.InstallMode {
+		out.Steps = s.setupSteps(ctx)
+	}
+	return &struct{ Body SetupState }{out}, nil
+}
+
+func setupDefaults() SetupDefaults {
+	d := formDefaults()
+	return SetupDefaults{BreakGlassUser: "admin", MinSignInLength: auth.MinBreakGlassPassword, OIDCSecretName: OIDCSecretName,
+		Timezone: d.Timezone, Retention: d.Retention, PollInterval: d.PollerInterval}
+}
+
+// setupSteps counts what exists. A failed read counts as nothing: it only decides which step the UI opens on.
+func (s *server) setupSteps(ctx context.Context) SetupSteps {
+	var st SetupSteps
+	if _, total, err := s.Store.ListDevices(ctx, store.DeviceFilter{}, store.Page{Limit: 1}); err == nil {
+		st.Devices = int(total)
+	}
+	if ps, err := s.Store.ListAlertPolicies(ctx); err == nil {
+		st.Policies = len(ps)
+	}
+	if f, err := s.formAdmin(); err == nil {
+		cfg, _ := f.CurrentConfig()
+		st.Receivers = len(cfg.Alerting.Receivers)
+		st.SignIn = cfg.Auth.OIDC != nil || cfg.Auth.BreakGlassFile != ""
+	}
+	return st
+}
+
+// setupErr maps the app's setup errors to problems.
+func (s *server) setupErr(err error) error {
+	switch {
+	case errors.Is(err, ErrSetupClosed):
+		return huma.Error409Conflict("setup is closed: it was completed, or a sign-in method was configured before first start")
+	case errors.Is(err, ErrSetupNoSignIn):
+		return huma.Error409Conflict("set up a sign-in method first: setup cannot finish without one")
+	case errors.Is(err, ErrInvalidSettings):
+		return settingsProblem(trimSentinel(err, ErrInvalidSettings), err)
+	}
+	s.Logger.Error("setup", "err", err)
+	return huma.Error500InternalServerError("setup failed; see the log")
+}
+
+func (s *server) setupSignIn(ctx context.Context, in *struct{ Body SetupSignInBody }) (*struct{ Body SetupSignIn }, error) {
+	a, err := s.setupAdmin()
+	if err != nil {
+		return nil, err
+	}
+	out, err := a.SignIn(ctx, actorOf(ctx), in.Body)
+	if err != nil {
+		return nil, s.setupErr(err)
+	}
+	return &struct{ Body SetupSignIn }{out}, nil
+}
+
+func (s *server) setupComplete(ctx context.Context, _ *struct{}) (*struct{ Body SetupComplete }, error) {
+	a, err := s.setupAdmin()
+	if err != nil {
+		return nil, err
+	}
+	out, err := a.Complete(ctx, actorOf(ctx))
+	if err != nil {
+		return nil, s.setupErr(err)
+	}
+	return &struct{ Body SetupComplete }{out}, nil
+}

internal/api/setup_policies.go

@@ -0,0 +1,63 @@
+package api
+
+import (
+	"context"
+
+	"github.com/v10networks/nmon/internal/alertpolicy"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// recommendedRules is the starter alert set: reachability, port errors and the three resource limits. Interface
+// down stays out on purpose (it is opt-in per port or group).
+var recommendedRules = []string{"device_down", "device_warning", "interface_errors", "cpu_high", "memory_high", "disk_full"}
+
+// recommendedName is the policy name for a rule.
+func recommendedName(rule string) string { return "recommended: " + rule }
+
+func (s *server) setupPolicies(ctx context.Context, _ *struct{ Body SetupPoliciesBody }) (*struct{ Body SetupPolicies }, error) {
+	a, err := s.setupAdmin()
+	if err != nil {
+		return nil, err
+	}
+	if !a.InstallMode() {
+		return nil, s.setupErr(ErrSetupClosed)
+	}
+	existing, err := s.Store.ListAlertPolicies(ctx)
+	if err != nil {
+		return nil, s.storeErr(err)
+	}
+	out := SetupPolicies{Created: []string{}, Existing: []string{}}
+	for _, rule := range recommendedRules {
+		name := recommendedName(rule)
+		if policyExists(existing, name, rule) {
+			out.Existing = append(out.Existing, name)
+			continue
+		}
+		row, err := s.policyRow(AlertPolicyBody{Name: name, Rule: rule, Action: alertpolicy.ActionAlert,
+			Scope: alertpolicy.Scope{All: true}}, true)
+		if err != nil {
+			return nil, err
+		}
+		if err := s.Store.CreateAlertPolicy(ctx, &row); err != nil {
+			return nil, s.storeErr(err)
+		}
+		out.Created = append(out.Created, name)
+	}
+	if len(out.Created) > 0 {
+		s.AlertPolicy.Changed()
+		if err := s.Store.AppendAudit(ctx, actorOf(ctx), "setup.recommended-policies", "alert-policies", nil, out.Created); err != nil {
+			s.Logger.Warn("audit setup.recommended-policies", "err", err)
+		}
+	}
+	return &struct{ Body SetupPolicies }{out}, nil
+}
+
+// policyExists reports whether a policy has this name or already binds the rule.
+func policyExists(all []store.AlertPolicy, name, rule string) bool {
+	for _, p := range all {
+		if p.Name == name || p.Rule == rule {
+			return true
+		}
+	}
+	return false
+}

internal/api/setup_test.go

@@ -0,0 +1,95 @@
+package api_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/auth"
+)
+
+type fakeSetup struct {
+	install bool
+	done    bool
+}
+
+func (f *fakeSetup) InstallMode() bool                            { return f.install }
+func (f *fakeSetup) SetupCompleted(context.Context) (bool, error) { return f.done, nil }
+func (f *fakeSetup) SignIn(context.Context, string, api.SetupSignInBody) (api.SetupSignIn, error) {
+	return api.SetupSignIn{}, api.ErrSetupClosed
+}
+func (f *fakeSetup) Complete(context.Context, string) (api.SetupComplete, error) {
+	return api.SetupComplete{}, api.ErrSetupNoSignIn
+}
+
+func TestSetupStateIsPublicAndCarriesNoSecrets(t *testing.T) {
+	fs := &fakeSetup{install: true}
+	e := newAuthedEnv(t, func(d *api.Deps) { d.Setup = fs })
+	rec := httptest.NewRecorder()
+	e.top.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/setup", nil))
+	if rec.Code != http.StatusOK {
+		t.Fatalf("GET /setup without credentials = %d %s", rec.Code, rec.Body)
+	}
+	var s api.SetupState
+	if err := json.Unmarshal(rec.Body.Bytes(), &s); err != nil || !s.InstallMode || s.Defaults.BreakGlassUser != "admin" ||
+		s.Defaults.MinSignInLength != auth.MinBreakGlassPassword || s.Defaults.PollInterval != "5m" {
+		t.Fatalf("state = %s (%v)", rec.Body, err)
+	}
+	if strings.Contains(strings.ToLower(rec.Body.String()), "password\":") {
+		t.Fatalf("state leaks a password field: %s", rec.Body)
+	}
+}
+
+func TestSetupPostsAnswer409WhenSetupIsClosed(t *testing.T) {
+	e := newEnvDeps(t, 0, func(d *api.Deps) { d.Setup = &fakeSetup{done: true}; d.AlertPolicy = &fakePolicyHost{} })
+	isProblem(t, e.do("POST", "/api/v1/setup/signin", `{"username":"admin","password":"a long enough one"}`), http.StatusConflict)
+	isProblem(t, e.do("POST", "/api/v1/setup/recommended-policies", `{}`), http.StatusConflict)
+	isProblem(t, e.do("POST", "/api/v1/setup/complete", ``), http.StatusConflict)
+}
+
+// allRulesHost knows every rule the recommended set binds.
+type allRulesHost struct{ fakePolicyHost }
+
+func (allRulesHost) Rules() []api.AlertRuleInfo {
+	var out []api.AlertRuleInfo
+	for name, level := range map[string]string{"device_down": "device", "device_warning": "device", "interface_errors": "interface",
+		"cpu_high": "device", "memory_high": "device", "disk_full": "component"} {
+		out = append(out, api.AlertRuleInfo{Name: name, Level: level, Severity: "warning"})
+	}
+	return out
+}
+
+func TestRecommendedPoliciesAreCreatedOnceAndIdempotently(t *testing.T) {
+	host := &allRulesHost{}
+	e := newEnvDeps(t, 0, func(d *api.Deps) { d.Setup = &fakeSetup{install: true}; d.AlertPolicy = host })
+	var first, second api.SetupPolicies
+	rec := e.do("POST", "/api/v1/setup/recommended-policies", `{}`)
+	if rec.Code != http.StatusOK || json.Unmarshal(rec.Body.Bytes(), &first) != nil || len(first.Created) != 6 {
+		t.Fatalf("first run: %d %s", rec.Code, rec.Body)
+	}
+	rec = e.do("POST", "/api/v1/setup/recommended-policies", `{}`)
+	if rec.Code != http.StatusOK || json.Unmarshal(rec.Body.Bytes(), &second) != nil || len(second.Created) != 0 || len(second.Existing) != 6 {
+		t.Fatalf("second run must create nothing: %d %s", rec.Code, rec.Body)
+	}
+	var list api.AlertPolicyList
+	e.getJSON(t, "/api/v1/alert-policies", &list)
+	if len(list.Items) != 6 || !list.Items[0].Scope.All || !list.Items[0].Enabled {
+		t.Fatalf("policies = %+v", list)
+	}
+}
+
+func TestInstallRoutesAreRealRoutes(t *testing.T) {
+	have := map[string]bool{}
+	for _, r := range api.Routes() {
+		have[r.Pattern] = true
+	}
+	for _, pat := range api.InstallRoutes() {
+		if !have[pat] {
+			t.Errorf("install mode allows %q, which is not an API route", pat)
+		}
+	}
+}

internal/api/setup_types.go

@@ -0,0 +1,69 @@
+package api
+
+// The first-install wizard's wire types. None of them carries a credential back to the caller: the password and
+// the OIDC client secret are write-only, like every other secret in the API.
+
+// SetupSteps says how far the wizard's work has got, so a reload of the page resumes in the right place.
+type SetupSteps struct {
+	SignIn    bool `json:"sign_in" doc:"A sign-in method (break-glass or OIDC) is configured."`
+	Devices   int  `json:"devices" doc:"Devices in the inventory."`
+	Receivers int  `json:"receivers" doc:"Configured notification receivers."`
+	Policies  int  `json:"policies" doc:"Alert policies."`
+}
+
+// SetupDefaults are the values the wizard prefills.
+type SetupDefaults struct {
+	BreakGlassUser  string        `json:"break_glass_user"`
+	MinSignInLength int           `json:"min_signin_length" doc:"Shortest break-glass password."`
+	OIDCSecretName  string        `json:"oidc_secret_name" doc:"Where the OIDC client secret is stored."`
+	Timezone        string        `json:"timezone"`
+	Retention       RetentionForm `json:"retention"`
+	PollInterval    string        `json:"poll_interval" doc:"Default poll interval, 30s to 1h."`
+}
+
+// SetupState is GET /setup.
+type SetupState struct {
+	InstallMode bool          `json:"install_mode" doc:"Setup is open: no sign-in exists, setup never completed, and callers are admins."`
+	Completed   bool          `json:"completed" doc:"Setup was completed. Install mode can never return."`
+	Steps       SetupSteps    `json:"steps" doc:"Filled in only while install mode is on."`
+	Defaults    SetupDefaults `json:"defaults"`
+}
+
+// SetupOIDC is the optional OIDC block of the sign-in step.
+type SetupOIDC struct {
+	Issuer       string `json:"issuer" minLength:"1"`
+	ClientID     string `json:"client_id" minLength:"1"`
+	ClientSecret string `json:"client_secret" minLength:"1" maxLength:"4096" doc:"Write-only. Stored encrypted under the default secret name."`
+	RedirectURL  string `json:"redirect_url" minLength:"1" doc:"nmon's callback, .../auth/callback."`
+	GroupsClaim  string `json:"groups_claim,omitempty"`
+	AdminGroup   string `json:"admin_group,omitempty"`
+}
+
+// SetupSignInBody is the body of POST /setup/signin.
+type SetupSignInBody struct {
+	Username string     `json:"username" minLength:"1" maxLength:"64" doc:"The break-glass admin. No colon or whitespace."`
+	Password string     `json:"password" minLength:"1" maxLength:"1024" doc:"Write-only. At least 12 characters, at most 72 bytes."`
+	OIDC     *SetupOIDC `json:"oidc,omitempty" doc:"Optional identity provider. Absent clears any OIDC set by an earlier attempt."`
+}
+
+// SetupSignIn is what the sign-in step did.
+type SetupSignIn struct {
+	BreakGlassUser string `json:"break_glass_user"`
+	OIDC           bool   `json:"oidc" doc:"OIDC was configured."`
+}
+
+// SetupPoliciesBody is the body of POST /setup/recommended-policies.
+type SetupPoliciesBody struct {
+	Receiver string `json:"receiver,omitempty" doc:"Informational: the receiver just set up. Receivers get every notification."`
+}
+
+// SetupPolicies lists what the recommended set did.
+type SetupPolicies struct {
+	Created  []string `json:"created" doc:"Policy names created now."`
+	Existing []string `json:"existing" doc:"Policy names skipped because a policy of that name, or for that rule, already exists."`
+}
+
+// SetupComplete is the outcome of POST /setup/complete.
+type SetupComplete struct {
+	SignInURL string `json:"sign_in_url" doc:"Where the browser goes to sign in: /auth/login with OIDC, else /auth/breakglass."`
+}

internal/api/viewer_test.go

@@ -141,6 +141,9 @@ func TestViewerAndPushCredentialsOnIngestAndAuthRoutes(t *testing.T) {
 func TestEveryRouteIsEitherPublicOrNeedsACredential(t *testing.T) {
 	e := newAuthedEnv(t, nil)
 	for _, r := range api.Routes() {
+		if r.Pattern == "GET /api/v1/setup" {
+			continue // public on purpose: the UI asks it before anyone can sign in (TestSetupStateIsPublicAndCarriesNoSecrets)
+		}
 		req := httptest.NewRequest(r.Method, concrete(r.Path), strings.NewReader("{}"))
 		rec := httptest.NewRecorder()
 		e.top.ServeHTTP(rec, req)

internal/app/alerting.go

@@ -13,6 +13,7 @@ import (
 	"github.com/v10networks/nmon/internal/config"
 	"github.com/v10networks/nmon/internal/netpolicy"
 	"github.com/v10networks/nmon/internal/notify"
+	"github.com/v10networks/nmon/internal/profile"
 	"github.com/v10networks/nmon/internal/reach"
 	"github.com/v10networks/nmon/internal/selfmon"
 	"github.com/v10networks/nmon/internal/store"
@@ -32,6 +33,7 @@ type alerting struct {
 	maint    *maintenanceCache
 	optIn    *optInSet
 	policies *policyEngine
+	registry *ruleRegistry
 
 	nt      *notifier
 	sys     *sysAlarms
@@ -57,11 +59,18 @@ func (a *App) newAlerting(s *settings) error {
 	n := newNotifier(q, s.alerting.Receivers, a.st, devices, a.log)
 	maint := newMaintenanceCache(a.st, clk, a.log)
 	optIn := newOptInSet(a.st, clk, entityTTL, a.log)
-	pe := newPolicyEngine(a.st, clk, optIn, s.alerting.Rules, a.log)
-	a.alerting = &alerting{queue: q, maint: maint, devices: devices, optIn: optIn, policies: pe}
+	reg := newRuleRegistry(a.st, clk, s.alerting.Rules, s.PollInterval, vmsup.NewQuerier(a.sup),
+		func() *profile.Set { return a.profiles }, a.log)
+	reg.cadence = a.cadence
+	pe := newPolicyEngine(a.st, clk, optIn, reg, a.log)
+	a.alerting = &alerting{queue: q, maint: maint, devices: devices, optIn: optIn, policies: pe, registry: reg}
+	if len(s.cfg.Alerting.Rules) > 0 {
+		a.log.Warn("alerting.rules in config.yaml is deprecated: define rules under Alerts > Alert rules instead; "+
+			"the configured rules still work", "count", len(s.cfg.Alerting.Rules))
+	}
 	a.alerting.reach = reach.New(reach.Options{
 		Store: a.st, Notifier: reachNotifier{n}, Clock: clk, Log: a.log, Wants: pe.Wants,
-		PollInterval: s.PollInterval, ICMPOff: s.ICMP == config.ICMPOff,
+		PollInterval: s.PollInterval, Cadence: a.cadence, ICMPOff: s.ICMP == config.ICMPOff,
 		ICMPBlocked: func(d *store.Device) bool { return a.icmpBlocked(d) },
 		InMaintenance: func(slug string, _ time.Time) bool {
 			_, ok := maint.Covered(slug)
@@ -98,7 +107,8 @@ func (a *App) startAlerting(s *settings) error {
 	al := a.alerting
 	var err error
 	al.rules, err = alert.New(alert.Options{
-		Store: a.st, Querier: vmsup.NewQuerier(a.sup), Notifier: alertNotifier{al.nt}, Clock: clock.Real(), Rules: s.alerting.Rules,
+		Store: a.st, Querier: vmsup.NewQuerier(a.sup), Notifier: alertNotifier{al.nt}, Clock: clock.Real(),
+		Rules: s.alerting.Rules, RuleSet: al.registry.Set,
 		Groups: groupLookup{a.st}, Unreliable: a.unreliable(), Policy: al.policies.Metric, Active: a.entityActive(), Suppressed: a.suppressed(),
 		StoreHealth: storeHealth(), Interval: s.alerting.Interval, Logger: a.log,
 	})

internal/app/apiwire.go

@@ -20,15 +20,15 @@ const deleteTimeout = time.Minute
 // apiDeps hands the API everything it needs.
 func (a *App) apiDeps(s *settings) api.Deps {
 	return api.Deps{
-		Store: a.st, Querier: vmsup.NewQuerier(a.sup), Poller: a.poller, SNMPPort: s.SNMPPort, Logger: a.log,
-		Profiles: a.profiles, PollerStatus: a.poller.Status, AfterDiscover: a.afterDiscover, PollInterval: s.PollInterval,
+		Store: a.st, Querier: vmsup.NewQuerier(a.sup), Poller: a.poller, PollTools: a.poller, SNMPPort: s.SNMPPort, Logger: a.log,
+		Profiles: a.profiles, PollerStatus: a.poller.Status, AfterDiscover: a.afterDiscover, PollInterval: s.PollInterval, Cadence: a.cadence,
 		HotRetention: s.cfg.Retention.Hot.D(), FiveMinuteRetention: s.cfg.Retention.FiveMinute.D(), LongRetention: s.cfg.Retention.Long.D(),
 		Reachability: a.alerting.reach, DeviceWarnings: a.alerting.reach.Warnings,
 		Dependencies: depGraph{a.alerting.reach}, OnMaintenanceChange: a.alerting.maint.Refresh,
-		OnAlertDownChange: a.alertDownChanged, AlertPolicy: a.alerting.policies,
+		OnAlertDownChange: a.alertDownChanged, AlertPolicy: a.alerting.policies, AlertRules: a.alerting.registry,
 		Tokens: a.svc.tokens, Secrets: a.st, Rollup: a.svc.rollup, P95: a.svc.p95, Groups: a.svc.groups, Zones: a.svc.zones,
 		Checks: a.svc.checks, Proxmox: a.svc.proxmox,
-		APITokens: a.auth.Tokens, Settings: a, Backups: a.backups, Stores: a.sup,
+		APITokens: a.curAuth().Tokens, Settings: a, Setup: a, Notify: a.alerting.queue, Backups: a.backups, Stores: a.sup,
 		Series: seriesDeleter{a.sup}, PurgeAfter: func() time.Duration { return a.live.get().PurgeAfter() },
 		OnImported: a.discoverNew, Vendors: a.vendors(), L2EdgeMax: a.l2EdgeMax, L2Health: a.l2Health, HealthNotes: a.healthNotes,
 	}
@@ -57,31 +57,66 @@ func (d seriesDeleter) DeleteSeries(ctx context.Context, store string, matchers
 	return vmsup.DeleteSeries(ctx, &http.Client{Timeout: deleteTimeout}, d.sup, store, matchers)
 }
 
-// startAPI binds the listen address and serves the API, the UI and push ingest, over TLS unless --insecure-http.
+// startAPI binds the listen address(es) and serves the API, the UI and push ingest: HTTPS on --listen (plain HTTP with
+// --insecure-http), plus the opt-in --http-listen plain-HTTP listener for a reverse proxy.
 func (a *App) startAPI(s *settings) error {
 	ui, err := webui.Handler(webui.Dist(), a.mux(api.New(a.apiDeps(s))))
 	if err != nil {
 		return err
 	}
-	ui = a.auth.Middleware(ui)
-	ln, err := net.Listen("tcp", s.Listen)
+	ui = a.authMiddleware(ui)
+	name, addr := "https", s.Listen
+	switch {
+	case a.tr.httpOnly: // no TLS configured: the proxy listener is the only one and --listen is not bound
+		name, addr = "http", s.HTTPListen
+	case a.tr.cfg == nil: // --insecure-http
+		name = "http"
+	}
+	if err := a.serveAPI(ui, name, addr, a.tr.cfg, true); err != nil {
+		return err
+	}
+	if a.tr.httpAddr != "" && !a.tr.httpOnly {
+		if err := a.serveAPI(ui, "http", a.tr.httpAddr, nil, false); err != nil {
+			return err
+		}
+	}
+	warnNoTrustedProxies(a.log, a.tr, s.cfg.Auth)
+	warnSelfSigned(a.log, a.tr, s.DataDir)
+	a.log.Info("serving", "transport", a.tr.describe())
+	a.startChallenge(s)
+	return nil
+}
+
+// serveAPI binds addr and serves h on it, with TLS when cfg is set. The primary listener sets a.ln (Addr); the
+// extra plain-HTTP one sets a.httpLn. Each registers its own shutdown, so they stop in reverse order.
+func (a *App) serveAPI(h http.Handler, name, addr string, cfg *tls.Config, primary bool) error {
+	ln, err := net.Listen("tcp", addr)
 	if err != nil {
-		return fmt.Errorf("listen: %w", err)
+		return fmt.Errorf("listen %s: %w", addr, err)
+	}
+	if primary {
+		a.ln = ln
+	} else {
+		a.httpLn = ln
 	}
-	a.ln = ln
 	serve := ln
-	if a.tr.cfg != nil {
-		serve = tls.NewListener(ln, a.tr.cfg)
+	if cfg != nil {
+		serve = tls.NewListener(ln, cfg)
+	}
+	srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
+	if primary {
+		a.srv = srv
 	}
-	a.srv = &http.Server{Handler: ui, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
 	go func() {
-		if err := a.srv.Serve(serve); !errors.Is(err, http.ErrServerClosed) {
-			a.serveErr <- err
+		if err := srv.Serve(serve); !errors.Is(err, http.ErrServerClosed) {
+			select {
+			case a.serveErr <- err:
+			default: // the first failure is the one reported
+			}
 		}
 	}()
-	a.onStop("http", a.srv.Shutdown)
-	a.log.Info("serving", "addr", ln.Addr().String(), "transport", a.tr.describe())
-	a.startChallenge(s)
+	a.onStop(name, srv.Shutdown)
+	a.log.Info("listening", "addr", ln.Addr().String(), "scheme", name)
 	return nil
 }
 
@@ -147,4 +182,5 @@ func (a *App) OnSIGHUP() {
 func (a *App) alertDownChanged() {
 	a.alerting.optIn.invalidate()
 	a.alerting.policies.Changed()
+	a.alerting.registry.Changed()
 }

internal/app/app.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"os/signal"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -54,6 +55,7 @@ type App struct {
 	poller *poller.Poller
 	srv    *http.Server
 	ln     net.Listener
+	httpLn net.Listener // the --http-listen listener when it runs beside HTTPS; nil otherwise
 
 	profiles *profile.Set
 	sched    *discovery.Scheduler
@@ -69,8 +71,15 @@ type App struct {
 
 	dataDir, keyFile string
 	live             *liveConfig // the configuration as running; see reload.go
+	settingsMu       sync.Mutex  // serialises settings writes (YAML, typed form, SIGHUP) so none overwrites another's read
 	tr               *transport
-	auth             *auth.Auth
+	flags            config.Flags              // the process flags, kept for settings that a flag overrides (poller.interval)
+	cadence          *store.PollCadence        // the live default poll interval
+	authp            atomic.Pointer[auth.Auth] // the live authentication; setup completion swaps it (install.go)
+	authMu           sync.Mutex
+	authRunning      *authRun
+	install          atomic.Bool // install mode is on
+	setupMu          sync.Mutex  // serialises the setup steps
 	guard            *diskguard.Guard
 	hotW             metrics.Writer // a.hot behind the disk guard's new-series admission
 	backups          *backupService
@@ -142,7 +151,8 @@ func Start(ctx context.Context, f config.Flags) (_ *App, err error) {
 	if err != nil {
 		return nil, err
 	}
-	a := &App{log: log, serveErr: make(chan error, 1), dataDir: s.DataDir, keyFile: s.keyFile, live: newLiveConfig(s.cfg), tr: s.tr}
+	a := &App{log: log, serveErr: make(chan error, 1), dataDir: s.DataDir, keyFile: s.keyFile, live: newLiveConfig(s.cfg), tr: s.tr,
+		flags: s.Flags, cadence: store.NewPollCadence(config.EffectivePollInterval(s.Flags, s.cfg))}
 	a.backups = &backupService{a: a}
 	defer func() {
 		if err != nil {
@@ -154,6 +164,9 @@ func Start(ctx context.Context, f config.Flags) (_ *App, err error) {
 	if err = a.startStore(ctx, s, kr); err != nil {
 		return nil, err
 	}
+	if err = a.decideAuth(ctx, s); err != nil {
+		return nil, err
+	}
 	if err = a.startMetricStores(ctx, s); err != nil {
 		return nil, err
 	}
@@ -206,6 +219,17 @@ func (a *App) startJobs(s *settings) error {
 // Addr is the address the API listens on.
 func (a *App) Addr() string { return a.ln.Addr().String() }
 
+// HTTPAddr is the address of the --http-listen listener, or "" when it is off. With plain HTTP only it equals Addr.
+func (a *App) HTTPAddr() string {
+	switch {
+	case a.httpLn != nil:
+		return a.httpLn.Addr().String()
+	case a.tr.httpOnly:
+		return a.ln.Addr().String()
+	}
+	return ""
+}
+
 // HotURL is the base URL of the hot metric store.
 func (a *App) HotURL() string { return a.sup.HotURL() }
 

internal/app/app_test.go

@@ -19,10 +19,9 @@ func TestStartRefusals(t *testing.T) {
 		f    config.Flags
 		want string
 	}{
-		"no insecure flag": {config.Flags{DataDir: data, SecretKeyFile: outside}, "--insecure-http"},
-		"no data dir":      {config.Flags{InsecureHTTP: true, SecretKeyFile: outside}, "--data-dir"},
-		"key inside data":  {config.Flags{DataDir: data, SecretKeyFile: filepath.Join(data, "keys", "secret.key"), InsecureHTTP: true}, "inside --data-dir"},
-		"key is data dir":  {config.Flags{DataDir: data, SecretKeyFile: data, InsecureHTTP: true}, "inside --data-dir"},
+		"no data dir":     {config.Flags{InsecureHTTP: true, SecretKeyFile: outside}, "--data-dir"},
+		"key inside data": {config.Flags{DataDir: data, SecretKeyFile: filepath.Join(data, "keys", "secret.key"), InsecureHTTP: true}, "inside --data-dir"},
+		"key is data dir": {config.Flags{DataDir: data, SecretKeyFile: data, InsecureHTTP: true}, "inside --data-dir"},
 	} {
 		t.Run(name, func(t *testing.T) {
 			a, err := Start(ctx, tc.f)

internal/app/authwire.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/v10networks/nmon/internal/api"
 	"github.com/v10networks/nmon/internal/auth"
+	"github.com/v10networks/nmon/internal/config"
 	"github.com/v10networks/nmon/internal/store"
 )
 
@@ -26,46 +27,81 @@ func breakGlassAlarmTTL(kind string) time.Duration {
 
 // startAuth builds the authentication service and declares the route policy. OIDC needs its client secret, a
 // named secret set through PUT /api/v1/secrets/{name}; while it is missing OIDC stays off (with a warning) and
-// break-glass still works, so an admin can sign in and set it.
+// break-glass still works, so an admin can sign in and set it. In install mode (no sign-in configured yet) the
+// service lets the setup wizard through and nothing else; see install.go.
 func (a *App) startAuth(ctx context.Context, s *settings) error {
-	ac := s.cfg.Auth
+	svc, err := a.newAuth(ctx, s.cfg.Auth, s.mode)
+	if err != nil {
+		return err
+	}
+	a.authp.Store(svc)
+	a.runAuth(svc)
+	a.onStop("auth", func(context.Context) error { a.stopAuthRun(); return nil })
+	a.logAuth(s, svc.OIDCEnabled())
+	return nil
+}
+
+// newAuth builds an auth service for ac in the given mode. It is used at start and again when setup completes,
+// so the live swap and a fresh start build exactly the same thing.
+func (a *App) newAuth(ctx context.Context, ac config.Auth, mode authMode) (*auth.Auth, error) {
 	opts := auth.Options{
 		Store: a.st, Alarms: kindAlarms{a.alerting.sys, breakGlassAlarmTTL}, Logger: a.log,
 		Session: auth.SessionConfig{
-			InsecureCookies: s.InsecureHTTP, Idle: ac.Session.Idle.D(), Absolute: ac.Session.Absolute.D(),
+			InsecureCookies: a.flags.InsecureHTTP, Idle: ac.Session.Idle.D(), Absolute: ac.Session.Absolute.D(),
 		},
 		BreakGlass:     auth.BreakGlassConfig{File: ac.BreakGlassFile, Username: ac.BreakGlassUser},
 		TrustedProxies: ac.TrustedProxies, IngestPrefixes: ingestPrefixes, Policy: auth.NewPolicy(),
-		DevOpen: s.mode.devOpen,
+		DevOpen: mode.devOpen, Install: mode.install,
 	}
-	oidc, err := a.oidcConfig(ctx, s)
+	oidc, err := a.oidcConfig(ctx, ac)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	opts.OIDC = oidc
 	svc, err := auth.New(ctx, opts)
 	if err != nil {
-		return fmt.Errorf("auth: %w", err)
+		return nil, fmt.Errorf("auth: %w", err)
 	}
 	api.DeclarePolicy(svc.Policy())
 	svc.Policy().Public("GET /healthz")
-	a.auth = svc
-	actx, cancel := context.WithCancel(context.Background())
-	done := make(chan struct{})
-	go func() { svc.Run(actx); close(done) }()
-	a.onStop("auth", func(context.Context) error {
-		cancel()
-		<-done
-		return nil
-	})
-	a.logAuth(s, oidc != nil)
-	return nil
+	return svc, nil
 }
 
+// authRun is the housekeeping goroutine of the current auth service.
+type authRun struct {
+	cancel context.CancelFunc
+	done   chan struct{}
+}
+
+// runAuth starts svc's housekeeping and remembers how to stop it.
+func (a *App) runAuth(svc *auth.Auth) {
+	actx, cancel := context.WithCancel(context.Background())
+	r := &authRun{cancel: cancel, done: make(chan struct{})}
+	go func() { svc.Run(actx); close(r.done) }()
+	a.authMu.Lock()
+	a.authRunning = r
+	a.authMu.Unlock()
+}
+
+// stopAuthRun stops the current auth service's housekeeping and waits for it.
+func (a *App) stopAuthRun() {
+	a.authMu.Lock()
+	r := a.authRunning
+	a.authRunning = nil
+	a.authMu.Unlock()
+	if r != nil {
+		r.cancel()
+		<-r.done
+	}
+}
+
+// curAuth is the live auth service; setup completion swaps it.
+func (a *App) curAuth() *auth.Auth { return a.authp.Load() }
+
 // oidcConfig reads the client secret from the secret store. nil means OIDC is not configured, or its secret
 // is not stored yet.
-func (a *App) oidcConfig(ctx context.Context, s *settings) (*auth.OIDCConfig, error) {
-	o := s.cfg.Auth.OIDC
+func (a *App) oidcConfig(ctx context.Context, ac config.Auth) (*auth.OIDCConfig, error) {
+	o := ac.OIDC
 	if o == nil {
 		return nil, nil
 	}
@@ -85,9 +121,13 @@ func (a *App) oidcConfig(ctx context.Context, s *settings) (*auth.OIDCConfig, er
 }
 
 func (a *App) logAuth(s *settings, oidc bool) {
-	if s.mode.devOpen {
+	switch {
+	case s.mode.devOpen:
 		warnDevOpen(a.log, s.Listen)
 		return
+	case s.mode.install:
+		a.startInstallWarnings()
+		return
 	}
 	a.log.Info("authentication on", "oidc", oidc, "break_glass", s.cfg.Auth.BreakGlassFile != "",
 		"trusted_proxies", len(s.cfg.Auth.TrustedProxies))
@@ -102,7 +142,7 @@ func (a *App) mux(apiH http.Handler) http.Handler {
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		_, _ = w.Write([]byte("ok\n"))
 	})
-	mux.Handle("/auth/", a.auth.Handler())
+	mux.Handle("/auth/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { a.curAuth().Handler().ServeHTTP(w, r) }))
 	a.ingester.Mount(mux)
 	mux.Handle("/", apiH)
 	return mux

internal/app/httplisten_test.go

@@ -0,0 +1,121 @@
+package app
+
+import (
+	"context"
+	"crypto/tls"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/config"
+)
+
+func TestPlanTransportAndAuthForHTTPListen(t *testing.T) {
+	signIn := config.Auth{BreakGlassFile: "bg"}
+	for name, tc := range map[string]struct {
+		f       config.Flags
+		auth    config.Auth
+		wantErr string // "" = success
+		httpAdr string
+		only    bool
+		desc    string
+	}{
+		"https only":                 {f: config.Flags{ACMEHosts: []string{"a.example"}}, auth: signIn, desc: "HTTPS with ACME certificates"},
+		"http only":                  {f: config.Flags{HTTPListen: "[::1]:8080"}, auth: signIn, httpAdr: "[::1]:8080", only: true, desc: "plain HTTP only"},
+		"https and http":             {f: config.Flags{ACMEHosts: []string{"a.example"}, HTTPListen: "[::1]:8080"}, auth: signIn, httpAdr: "[::1]:8080", desc: "and plain HTTP on [::1]:8080"},
+		"http and insecure conflict": {f: config.Flags{HTTPListen: "[::1]:8080", InsecureHTTP: true}, auth: signIn, wantErr: "mutually exclusive"},
+		"http without sign-in":       {f: config.Flags{HTTPListen: "[::1]:8080"}, wantErr: "no way to sign in"},
+		"https+http without sign-in": {f: config.Flags{ACMEHosts: []string{"a.example"}, HTTPListen: "[::1]:8080"}, wantErr: "no way to sign in"},
+		"no tls flags: self-signed":  {f: config.Flags{}, auth: signIn, desc: "HTTPS with a self-signed certificate"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			tr, err := planTransport(&tc.f, t.TempDir())
+			if err == nil {
+				var m authMode
+				if m, err = planAuth(tr, tc.auth, true); err == nil && m.devOpen {
+					t.Fatal("--http-listen must never enable dev-open mode")
+				}
+			}
+			if tc.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
+					t.Fatalf("error %v, want one mentioning %q", err, tc.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if tr.httpAddr != tc.httpAdr || tr.httpOnly != tc.only || tr.insecure || !strings.Contains(tr.describe(), tc.desc) {
+				t.Fatalf("transport %+v (%s), want http=%q only=%v desc %q", tr, tr.describe(), tc.httpAdr, tc.only, tc.desc)
+			}
+		})
+	}
+}
+
+// postBreakGlass signs in over HTTP, with extra headers, and returns the session cookie's Secure attribute.
+func postBreakGlass(t *testing.T, base string, hdr map[string]string) (status int, secure bool) {
+	t.Helper()
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, base+"/auth/breakglass",
+		strings.NewReader(`{"username":"breakglass","password":"pw"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	for k, v := range hdr {
+		req.Header.Set(k, v)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	for _, c := range resp.Cookies() {
+		secure = secure || c.Secure
+	}
+	return resp.StatusCode, secure
+}
+
+func TestHTTPListenServesBesideHTTPSWithSignInAndProxyAwareCookies(t *testing.T) {
+	certFile, keyFile, pool := selfSigned(t, 1)
+	a, _ := startWith(t, func(f *config.Flags) {
+		f.InsecureHTTP, f.TLSCert, f.TLSKey, f.HTTPListen = false, certFile, keyFile, "[::1]:0"
+	}, "auth:\n  break_glass_file: "+breakGlassFile(t)+"\n  trusted_proxies: [\"::1\"]\n")
+	httpBase := "http://" + a.HTTPAddr()
+	if !strings.HasPrefix(a.HTTPAddr(), "[::1]:") || a.HTTPAddr() == a.Addr() {
+		t.Fatalf("listeners: https %s, http %s", a.Addr(), a.HTTPAddr())
+	}
+	tc := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
+	if code, _ := getBody(t, tc, "https://"+a.Addr()+"/healthz"); code != http.StatusOK {
+		t.Fatalf("https /healthz = %d", code)
+	}
+	if code, _ := getBody(t, http.DefaultClient, httpBase+"/healthz"); code != http.StatusOK {
+		t.Fatalf("http /healthz = %d", code)
+	}
+	if code, _ := getBody(t, http.DefaultClient, httpBase+"/api/v1/me"); code != http.StatusUnauthorized {
+		t.Fatalf("sign-in must stay on over the HTTP listener, /api/v1/me = %d", code)
+	}
+	// ::1 is the trusted proxy here: only its X-Forwarded-Proto: https makes the cookie Secure.
+	if code, secure := postBreakGlass(t, httpBase, nil); code != http.StatusNoContent || secure {
+		t.Fatalf("direct http sign-in: %d, Secure=%v (want 204, not Secure)", code, secure)
+	}
+	if code, secure := postBreakGlass(t, httpBase, map[string]string{"X-Forwarded-Proto": "https"}); code != http.StatusNoContent || !secure {
+		t.Fatalf("proxied https sign-in: %d, Secure=%v (want 204, Secure)", code, secure)
+	}
+}
+
+func TestHTTPListenAloneBindsOnlyTheHTTPListener(t *testing.T) {
+	a, _ := startWith(t, func(f *config.Flags) {
+		f.InsecureHTTP, f.HTTPListen, f.Listen = false, "[::1]:0", "127.0.0.1:1" // --listen must not be bound
+	}, "auth:\n  break_glass_file: "+breakGlassFile(t)+"\n")
+	if a.HTTPAddr() != a.Addr() || !strings.HasPrefix(a.Addr(), "[::1]:") {
+		t.Fatalf("addr %s, http addr %s", a.Addr(), a.HTTPAddr())
+	}
+	base := "http://" + a.Addr()
+	if code, _ := getBody(t, http.DefaultClient, base+"/api/v1/me"); code != http.StatusUnauthorized {
+		t.Fatalf("/api/v1/me = %d, want 401 (not dev-open)", code)
+	}
+	// No trusted proxy configured: a spoofed X-Forwarded-Proto is ignored, so the cookie is not Secure.
+	if code, secure := postBreakGlass(t, base, map[string]string{"X-Forwarded-Proto": "https"}); code != http.StatusNoContent || secure {
+		t.Fatalf("sign-in: %d, Secure=%v", code, secure)
+	}
+}

internal/app/install.go

@@ -0,0 +1,201 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/auth"
+	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/config"
+)
+
+// Install mode is the first-install wizard's window. It is open only while no sign-in method is configured and
+// setup was never completed; the stored flag (store.SetupState) makes the end permanent, so editing config.yaml or
+// deleting the break-glass file later cannot reopen it. There is no setup token and no localhost restriction by
+// owner decision: nmon is deployed inside a DMZ or an internal network, and whoever reaches it in install mode can
+// run setup (DEVIATIONS 51). The auth service enforces the allow-list (auth.Policy.AllowInstall).
+
+// installWarnEvery is how often the "setup is open" warning repeats while install mode lasts.
+const installWarnEvery = 10 * time.Minute
+
+// breakGlassFileName is the break-glass file the wizard writes inside the data dir.
+const breakGlassFileName = "breakglass"
+
+// decideAuth runs once the inventory is open. It reads the stored setup flag, applies planAuth, and marks an
+// existing install (one that already has a sign-in method) completed, so it can never enter install mode later.
+func (a *App) decideAuth(ctx context.Context, s *settings) error {
+	done, _, err := a.st.SetupCompleted(ctx)
+	if err != nil {
+		return fmt.Errorf("read setup state: %w", err)
+	}
+	mode, err := planAuth(s.tr, s.cfg.Auth, done)
+	if err != nil {
+		return err
+	}
+	if !done && signInConfigured(s.cfg.Auth) {
+		if err := a.st.MarkSetupCompleted(ctx, "startup: a sign-in method was already configured"); err != nil {
+			return fmt.Errorf("mark setup completed: %w", err)
+		}
+		a.log.Info("sign-in is already configured: setup marked completed, install mode can never open")
+	}
+	s.mode = mode
+	a.install.Store(mode.install)
+	return nil
+}
+
+// authMiddleware authenticates with whichever auth service is live, so the swap on setup completion applies to the
+// very next request.
+func (a *App) authMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { a.curAuth().Middleware(next).ServeHTTP(w, r) })
+}
+
+// startInstallWarnings logs the "setup is open" warning now and every installWarnEvery until install mode ends.
+func (a *App) startInstallWarnings() {
+	warn := func() {
+		a.log.Warn("INSTALL MODE: setup is open to anyone who can reach this address, and they are treated as an admin; "+
+			"open the web UI now and finish setup (sign-in comes first)", "mode", "install", "listen", a.listenAddr())
+	}
+	warn()
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		t := clock.Real().NewTicker(installWarnEvery)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C():
+				if !a.install.Load() {
+					return
+				}
+				warn()
+			}
+		}
+	}()
+	a.onStop("install warnings", func(context.Context) error { cancel(); <-done; return nil })
+}
+
+// listenAddr names where nmon is reachable, for the install warning.
+func (a *App) listenAddr() string {
+	if a.tr.httpOnly {
+		return a.tr.httpAddr
+	}
+	return a.flags.Listen
+}
+
+// InstallMode implements api.SetupAdmin.
+func (a *App) InstallMode() bool { return a.install.Load() }
+
+// SetupCompleted implements api.SetupAdmin.
+func (a *App) SetupCompleted(ctx context.Context) (bool, error) {
+	done, _, err := a.st.SetupCompleted(ctx)
+	return done, err
+}
+
+// SignIn implements api.SetupAdmin: break-glass file first (it validates the user and password), then the optional
+// OIDC secret, then auth.* in config.yaml through the ordinary settings save path.
+func (a *App) SignIn(ctx context.Context, actor string, in api.SetupSignInBody) (api.SetupSignIn, error) {
+	a.setupMu.Lock()
+	defer a.setupMu.Unlock()
+	if !a.install.Load() {
+		return api.SetupSignIn{}, api.ErrSetupClosed
+	}
+	user := strings.TrimSpace(in.Username)
+	if err := checkSetupOIDC(in.OIDC); err != nil {
+		return api.SetupSignIn{}, fmt.Errorf("%w: %w", api.ErrInvalidSettings, err)
+	}
+	file := filepath.Join(a.dataDir, breakGlassFileName)
+	if err := auth.WriteBreakGlassFile(file, user, in.Password); err != nil {
+		return api.SetupSignIn{}, fmt.Errorf("%w: password: %w", api.ErrInvalidSettings, err)
+	}
+	if o := in.OIDC; o != nil {
+		if err := a.st.PutSecret(ctx, api.OIDCSecretName, o.ClientSecret); err != nil {
+			return api.SetupSignIn{}, fmt.Errorf("store the OIDC client secret: %w", err)
+		}
+	}
+	_, err := a.UpdateConfig(ctx, actor, func(c *config.Config) error {
+		c.Auth.BreakGlassFile, c.Auth.BreakGlassUser = file, ""
+		c.Auth.OIDC = oidcConfigFor(in.OIDC)
+		return nil
+	})
+	if err != nil {
+		return api.SetupSignIn{}, err
+	}
+	if err := a.st.AppendAudit(ctx, actor, "setup.signin", "auth", nil, map[string]any{"user": user, "oidc": in.OIDC != nil}); err != nil {
+		a.log.Warn("audit setup.signin", "err", err)
+	}
+	return api.SetupSignIn{BreakGlassUser: user, OIDC: in.OIDC != nil}, nil
+}
+
+// oidcConfigFor is the config.yaml block for the wizard's OIDC fields, or nil without them.
+func oidcConfigFor(o *api.SetupOIDC) *config.OIDC {
+	if o == nil {
+		return nil
+	}
+	return &config.OIDC{Issuer: strings.TrimSpace(o.Issuer), ClientID: strings.TrimSpace(o.ClientID),
+		ClientSecretName: api.OIDCSecretName, RedirectURL: strings.TrimSpace(o.RedirectURL),
+		GroupsClaim: strings.TrimSpace(o.GroupsClaim), AdminGroup: strings.TrimSpace(o.AdminGroup)}
+}
+
+// checkSetupOIDC validates the OIDC fields before anything is written. Discovery is not contacted here: the
+// provider is only reached at the first sign-in, as it is for an OIDC block written by hand.
+func checkSetupOIDC(o *api.SetupOIDC) error {
+	if o == nil {
+		return nil
+	}
+	return config.Auth{OIDC: oidcConfigFor(o)}.Validate()
+}
+
+// Complete implements api.SetupAdmin. Order matters: the new auth service is built first (a failure leaves install
+// mode on), then the flag is stored (the point of no return), then the live service is swapped.
+func (a *App) Complete(ctx context.Context, actor string) (api.SetupComplete, error) {
+	a.setupMu.Lock()
+	defer a.setupMu.Unlock()
+	if !a.install.Load() {
+		return api.SetupComplete{}, api.ErrSetupClosed
+	}
+	cfg := a.live.get()
+	if !signInConfigured(cfg.Auth) {
+		return api.SetupComplete{}, api.ErrSetupNoSignIn
+	}
+	svc, err := a.newAuth(ctx, cfg.Auth, authMode{})
+	if err != nil {
+		return api.SetupComplete{}, fmt.Errorf("build authentication: %w", err)
+	}
+	if err := a.st.MarkSetupCompleted(ctx, actor); err != nil {
+		return api.SetupComplete{}, fmt.Errorf("mark setup completed: %w", err)
+	}
+	a.swapAuth(svc, cfg.Auth)
+	if err := a.st.AppendAudit(ctx, actor, "setup.complete", "setup", nil, nil); err != nil {
+		a.log.Warn("audit setup.complete", "err", err)
+	}
+	a.log.Info("setup completed: authentication is on, install mode is closed for good", "by", actor, "oidc", svc.OIDCEnabled())
+	url := "/auth/breakglass"
+	if svc.OIDCEnabled() {
+		url = "/auth/login"
+	}
+	return api.SetupComplete{SignInURL: url}, nil
+}
+
+// swapAuth makes svc the live authentication without a restart. Auth is already behind an atomic pointer that
+// the middleware, the /auth/ handler and the trusted-proxy reload read per request, so the swap is one store; the
+// old service's housekeeping is replaced by the new one's. The auth section is now what is running, so it no
+// longer counts as "restart required".
+func (a *App) swapAuth(svc *auth.Auth, ac config.Auth) {
+	a.stopAuthRun()
+	a.authp.Store(svc)
+	a.runAuth(svc)
+	a.install.Store(false)
+	a.live.mu.Lock()
+	a.live.running.Auth = ac
+	a.live.mu.Unlock()
+}
+
+var _ api.SetupAdmin = (*App)(nil)

internal/app/install_test.go

@@ -0,0 +1,234 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/crypto/bcrypt"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/config"
+)
+
+func TestPlanAuthDecisionTable(t *testing.T) {
+	signIn := config.Auth{BreakGlassFile: "bg"}
+	oidc := config.Auth{OIDC: &config.OIDC{Issuer: "https://idp.example"}}
+	insecure := &transport{insecure: true}
+	https := &transport{}
+	httpOnly := &transport{httpOnly: true, httpAddr: "[::1]:8080"}
+	for name, tc := range map[string]struct {
+		tr        *transport
+		auth      config.Auth
+		completed bool
+		want      authMode
+		wantErr   bool
+	}{
+		"fresh install over https":            {tr: https, want: authMode{install: true}},
+		"fresh install over --http-listen":    {tr: httpOnly, want: authMode{install: true}},
+		"break-glass configured":              {tr: https, auth: signIn},
+		"oidc configured":                     {tr: https, auth: oidc},
+		"sign-in configured and completed":    {tr: https, auth: signIn, completed: true},
+		"completed, then sign-in was removed": {tr: https, completed: true, wantErr: true},
+		"completed over --http-listen":        {tr: httpOnly, completed: true, wantErr: true},
+		"--insecure-http stays dev-open":      {tr: insecure, want: authMode{devOpen: true}},
+		"--insecure-http, setup completed":    {tr: insecure, completed: true, want: authMode{devOpen: true}},
+		"--insecure-http with sign-in":        {tr: insecure, auth: signIn},
+	} {
+		t.Run(name, func(t *testing.T) {
+			m, err := planAuth(tc.tr, tc.auth, tc.completed)
+			if (err != nil) != tc.wantErr || m != tc.want {
+				t.Fatalf("planAuth = %+v, %v; want %+v (error %v)", m, err, tc.want, tc.wantErr)
+			}
+		})
+	}
+}
+
+// installFlags starts nmon on plain HTTP with no sign-in: a fresh install, in install mode.
+func installFlags(f *config.Flags) { f.InsecureHTTP, f.HTTPListen = false, "[::1]:0" }
+
+func doJSON(t *testing.T, method, url, body string) (int, string) {
+	t.Helper()
+	req, err := http.NewRequestWithContext(context.Background(), method, url, strings.NewReader(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	b, _ := io.ReadAll(resp.Body)
+	return resp.StatusCode, string(b)
+}
+
+func TestInstallModeAllowsOnlyTheWizardRoutesAndAnswers503Elsewhere(t *testing.T) {
+	a, _ := startWith(t, installFlags, "")
+	if !a.InstallMode() {
+		t.Fatal("a fresh install with no sign-in must be in install mode")
+	}
+	base := "http://" + a.Addr()
+	for _, p := range []string{"/api/v1/me", "/api/v1/devices/x/overview", "/api/v1/tokens", "/api/v1/audit", "/api/v1/alerts", "/metrics"} {
+		code, body := getBody(t, http.DefaultClient, base+p)
+		if code != http.StatusServiceUnavailable || !strings.Contains(body, "setup not finished") {
+			t.Errorf("GET %s in install mode = %d %s, want 503 problem \"setup not finished\"", p, code, body)
+		}
+	}
+	for _, p := range []string{"/api/v1/setup", "/api/v1/meta", "/api/v1/devices", "/api/v1/settings/form", "/api/v1/notify/presets",
+		"/api/v1/profiles", "/api/v1/alert-policies", "/healthz"} {
+		if code, body := getBody(t, http.DefaultClient, base+p); code != http.StatusOK {
+			t.Errorf("GET %s in install mode = %d %s, want 200", p, code, body)
+		}
+	}
+	if code, _ := getBody(t, http.DefaultClient, base+"/"); code == http.StatusServiceUnavailable {
+		t.Error("the UI must be served in install mode")
+	}
+	if code, _ := doJSON(t, http.MethodPost, base+"/api/v1/devices/x/retire", "{}"); code != http.StatusServiceUnavailable {
+		t.Errorf("a state-changing route outside the wizard = %d, want 503", code)
+	}
+}
+
+func TestSignInThenCompleteEndsInstallModeForGoodAcrossRestarts(t *testing.T) {
+	a, f := startWith(t, installFlags, "")
+	base := "http://" + a.Addr()
+	if code, body := doJSON(t, http.MethodPost, base+"/api/v1/setup/complete", ""); code != http.StatusConflict {
+		t.Fatalf("complete before sign-in = %d %s, want 409", code, body)
+	}
+	if code, body := doJSON(t, http.MethodPost, base+"/api/v1/setup/signin", `{"username":"admin","password":"short"}`); code != http.StatusUnprocessableEntity {
+		t.Fatalf("short password = %d %s, want 422", code, body)
+	}
+	const pw = "correct horse battery"
+	if code, body := doJSON(t, http.MethodPost, base+"/api/v1/setup/signin", `{"username":"admin","password":"`+pw+`"}`); code != http.StatusOK {
+		t.Fatalf("sign-in step = %d %s", code, body)
+	} else if strings.Contains(body, pw) {
+		t.Fatal("the response echoes the password")
+	}
+	file := filepath.Join(f.DataDir, "breakglass")
+	if st, err := os.Stat(file); err != nil || st.Mode().Perm() != 0o600 {
+		t.Fatalf("break-glass file: %v %v", st, err)
+	}
+	raw, _ := os.ReadFile(file)
+	user, hash, _ := strings.Cut(strings.TrimSpace(string(raw)), ":")
+	if user != "admin" || bcrypt.CompareHashAndPassword([]byte(hash), []byte(pw)) != nil {
+		t.Fatalf("break-glass file %q is not user:bcrypt(password)", raw)
+	}
+	if cfg, err := config.Load(f.DataDir); err != nil || cfg.Auth.BreakGlassFile != file {
+		t.Fatalf("config.yaml auth.break_glass_file = %q (%v)", cfg.Auth.BreakGlassFile, err)
+	}
+	if !a.InstallMode() {
+		t.Fatal("install mode must last until complete")
+	}
+	code, body := doJSON(t, http.MethodPost, base+"/api/v1/setup/complete", "")
+	if code != http.StatusOK || !strings.Contains(body, "/auth/breakglass") {
+		t.Fatalf("complete = %d %s", code, body)
+	}
+	if a.InstallMode() {
+		t.Fatal("install mode still on after complete")
+	}
+	// Authentication is live without a restart: the same listener now demands a credential.
+	if code, _ := getBody(t, http.DefaultClient, base+"/api/v1/me"); code != http.StatusUnauthorized {
+		t.Fatalf("/api/v1/me after complete = %d, want 401", code)
+	}
+	if code, _ := doJSON(t, http.MethodPost, base+"/api/v1/setup/signin", `{"username":"x","password":"`+pw+`"}`); code != http.StatusUnauthorized {
+		t.Fatalf("setup POST after complete without a credential = %d, want 401", code)
+	}
+	if _, err := a.SignIn(context.Background(), "t", api.SetupSignInBody{Username: "x", Password: pw}); err == nil {
+		t.Fatal("sign-in step must be closed after completion")
+	}
+	var st struct{ InstallMode, Completed bool }
+	_, body = getBody(t, http.DefaultClient, base+"/api/v1/setup")
+	if err := json.Unmarshal([]byte(strings.NewReplacer("install_mode", "InstallMode", "completed", "Completed").Replace(body)), &st); err != nil ||
+		st.InstallMode || !st.Completed {
+		t.Fatalf("GET /setup after complete: %s", body)
+	}
+	stopApp(t, a)
+	restartAfterCompletion(t, f, file)
+}
+
+// restartAfterCompletion proves the flag is permanent: a restart is not in install mode, and removing the sign-in
+// from config.yaml and deleting the break-glass file does not reopen setup (nmon refuses to start instead).
+func restartAfterCompletion(t *testing.T, f config.Flags, file string) {
+	t.Helper()
+	installFlags(&f)
+	a2, err := Start(context.Background(), f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a2.InstallMode() {
+		t.Fatal("install mode came back after a restart")
+	}
+	if code, _ := getBody(t, http.DefaultClient, "http://"+a2.Addr()+"/api/v1/me"); code != http.StatusUnauthorized {
+		t.Fatalf("after restart /api/v1/me = %d, want 401", code)
+	}
+	stopApp(t, a2)
+	if err := os.Remove(file); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(config.File(f.DataDir), []byte("timezone: UTC\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	a3, err := Start(context.Background(), f)
+	if err == nil {
+		stopApp(t, a3)
+		t.Fatal("a completed install with no sign-in must refuse to start, not reopen setup")
+	}
+	if !strings.Contains(err.Error(), "no way to sign in") {
+		t.Fatalf("error %v", err)
+	}
+}
+
+func TestStartMarksAnExistingInstallWithSignInCompleted(t *testing.T) {
+	a, _ := startWith(t, installFlags, "auth:\n  break_glass_file: "+breakGlassFile(t)+"\n")
+	if a.InstallMode() {
+		t.Fatal("a configured sign-in means no install mode")
+	}
+	if done, _, err := a.st.SetupCompleted(context.Background()); err != nil || !done {
+		t.Fatalf("existing install not marked completed: %v %v", done, err)
+	}
+}
+
+func TestDevOpenIsNeverInstallMode(t *testing.T) {
+	a, _ := startWith(t, nil, "")
+	if a.InstallMode() {
+		t.Fatal("--insecure-http must stay dev-open, not install mode")
+	}
+	if done, _, _ := a.st.SetupCompleted(context.Background()); done {
+		t.Fatal("dev-open must not mark setup completed")
+	}
+}
+
+func TestPollerIntervalSettingReachesTheLiveCadenceAndTheFlagWins(t *testing.T) {
+	a, _ := startWith(t, nil, "poller:\n  interval: 2m\n")
+	if got := a.cadence.Get(); got != 2*time.Minute {
+		t.Fatalf("cadence at start = %v, want 2m", got)
+	}
+	set := func(v string) {
+		t.Helper()
+		_, err := a.UpdateConfig(context.Background(), "t", func(c *config.Config) error {
+			d, err := config.ParseDuration(v)
+			c.Poller.Interval = config.Duration(d)
+			return err
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	set("10m")
+	if got := a.cadence.Get(); got != 10*time.Minute {
+		t.Fatalf("cadence after reload = %v, want 10m", got)
+	}
+	if _, err := a.UpdateConfig(context.Background(), "t", func(c *config.Config) error { c.Poller.Interval = config.Duration(5 * time.Second); return nil }); err == nil {
+		t.Fatal("5s is below the 30s minimum")
+	}
+	b, _ := startWith(t, func(f *config.Flags) { f.PollInterval = 45 * time.Second }, "poller:\n  interval: 2m\n")
+	if got := b.cadence.Get(); got != 45*time.Second {
+		t.Fatalf("flag must win over config: %v", got)
+	}
+}

internal/app/lifecycle_test.go

@@ -235,8 +235,11 @@ func TestRunStopsWhenContextEnds(t *testing.T) {
 }
 
 func TestRunReturnsStartError(t *testing.T) {
-	if err := Run(context.Background(), config.Flags{DataDir: t.TempDir()}); err == nil {
-		t.Fatal("expected the --insecure-http refusal")
+	// No TLS flags is no longer an error (install mode serves a self-signed certificate), so use a flag combination
+	// that is always refused.
+	f := config.Flags{DataDir: t.TempDir(), TLSCert: "cert.pem"}
+	if err := Run(context.Background(), f); err == nil {
+		t.Fatal("expected the --tls-cert without --tls-key refusal")
 	}
 }
 

internal/app/m6_test.go

@@ -140,18 +140,16 @@ func TestPlanTransportAndAuthRefusals(t *testing.T) {
 		auth config.Auth
 		want string
 	}{
-		"no tls and no insecure":    {config.Flags{}, config.Auth{}, "--tls-cert"},
-		"half a certificate pair":   {config.Flags{TLSCert: "c.pem"}, config.Auth{}, "go together"},
-		"cert and acme":             {config.Flags{TLSCert: "c", TLSKey: "k", ACMEHosts: []string{"a.example"}}, config.Auth{}, "not both"},
-		"insecure with tls":         {config.Flags{InsecureHTTP: true, ACMEHosts: []string{"a.example"}}, config.Auth{}, "cannot be combined"},
-		"unreadable certificate":    {config.Flags{TLSCert: "/nonexistent/c", TLSKey: "/nonexistent/k"}, config.Auth{}, "tls: load"},
-		"acme needs a host":         {config.Flags{ACMEHosts: []string{}}, config.Auth{}, "--tls-cert"},
-		"tls and no sign-in method": {config.Flags{ACMEHosts: []string{"a.example"}}, config.Auth{}, "no way to sign in"},
+		"half a certificate pair":        {config.Flags{TLSCert: "c.pem"}, config.Auth{}, "go together"},
+		"cert and acme":                  {config.Flags{TLSCert: "c", TLSKey: "k", ACMEHosts: []string{"a.example"}}, config.Auth{}, "not both"},
+		"insecure with tls":              {config.Flags{InsecureHTTP: true, ACMEHosts: []string{"a.example"}}, config.Auth{}, "cannot be combined"},
+		"unreadable certificate":         {config.Flags{TLSCert: "/nonexistent/c", TLSKey: "/nonexistent/k"}, config.Auth{}, "tls: load"},
+		"tls, no sign-in and setup done": {config.Flags{ACMEHosts: []string{"a.example"}}, config.Auth{}, "no way to sign in"},
 	} {
 		t.Run(name, func(t *testing.T) {
 			tr, err := planTransport(&tc.f, t.TempDir())
 			if err == nil {
-				_, err = planAuth(tr, tc.auth)
+				_, err = planAuth(tr, tc.auth, true)
 			}
 			if err == nil || !strings.Contains(err.Error(), tc.want) {
 				t.Fatalf("error %v, want one mentioning %q", err, tc.want)
@@ -162,10 +160,10 @@ func TestPlanTransportAndAuthRefusals(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if m, err := planAuth(tr, config.Auth{}); err != nil || !m.devOpen {
+	if m, err := planAuth(tr, config.Auth{}, false); err != nil || !m.devOpen {
 		t.Fatalf("insecure-http with no sign-in must be dev-open: %+v %v", m, err)
 	}
-	if m, err := planAuth(tr, config.Auth{BreakGlassFile: "x"}); err != nil || m.devOpen {
+	if m, err := planAuth(tr, config.Auth{BreakGlassFile: "x"}, false); err != nil || m.devOpen {
 		t.Fatalf("a configured sign-in turns dev-open off: %+v %v", m, err)
 	}
 }

internal/app/policies.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/v10networks/nmon/internal/alert"
 	"github.com/v10networks/nmon/internal/alertpolicy"
+	"github.com/v10networks/nmon/internal/api"
 	"github.com/v10networks/nmon/internal/clock"
 	"github.com/v10networks/nmon/internal/store"
 )
@@ -17,6 +18,11 @@ import (
 // policyRefresh is how often the policy table is re-read; an API write invalidates it at once.
 const policyRefresh = 30 * time.Second
 
+// ruleLister names every rule a policy may bind, with its level.
+type ruleLister interface {
+	Rules() []api.AlertRuleInfo
+}
+
 // policyStore is what the policy engine reads.
 type policyStore interface {
 	store.AlertPolicies
@@ -33,7 +39,7 @@ type policyEngine struct {
 	st    policyStore
 	log   *slog.Logger
 	optIn *optInSet
-	rules []alert.Rule
+	rules ruleLister
 
 	set       *ttlCache[struct{}, *alertpolicy.Set]
 	devices   *ttlCache[string, store.Device]
@@ -50,7 +56,7 @@ type policyEngine struct {
 	warned  bool
 }
 
-func newPolicyEngine(st policyStore, clk clock.Clock, optIn *optInSet, rules []alert.Rule, log *slog.Logger) *policyEngine {
+func newPolicyEngine(st policyStore, clk clock.Clock, optIn *optInSet, rules ruleLister, log *slog.Logger) *policyEngine {
 	return &policyEngine{
 		st: st, log: log, optIn: optIn, rules: rules, clk: clk,
 		lastDev: map[string]store.Device{}, lastIf: map[string]store.Interface{},

internal/app/policies_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/v10networks/nmon/internal/alert"
 	"github.com/v10networks/nmon/internal/alertpolicy"
 	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/profile"
 	"github.com/v10networks/nmon/internal/store"
 	"github.com/v10networks/nmon/internal/testutil"
 	"github.com/v10networks/nmon/internal/ulid"
@@ -19,6 +20,7 @@ type policyEnv struct {
 	st  store.Store
 	clk *clock.Fake
 	pe  *policyEngine
+	reg *ruleRegistry
 	dev *store.Device
 }
 
@@ -32,7 +34,8 @@ func newPolicyEnv(t *testing.T) *policyEnv {
 		t.Fatal(err)
 	}
 	e := &policyEnv{t: t, st: st, clk: clk, dev: d}
-	e.pe = newPolicyEngine(st, clk, newOptInSet(st, clk, entityTTL, testLogger()), alert.DefaultRules(), testLogger())
+	e.reg = newRuleRegistry(st, clk, alert.DefaultRules(), time.Minute, nil, func() *profile.Set { return nil }, testLogger())
+	e.pe = newPolicyEngine(st, clk, newOptInSet(st, clk, entityTTL, testLogger()), e.reg, testLogger())
 	return e
 }
 

internal/app/policypreview.go

@@ -12,21 +12,8 @@ import (
 // previewSamples is how many covered devices and ports a preview names.
 const previewSamples = 20
 
-// Rules implements api.AlertPolicyHost: the metric rules plus the reachability and system rules.
-func (p *policyEngine) Rules() []api.AlertRuleInfo {
-	out := make([]api.AlertRuleInfo, 0, len(p.rules)+3)
-	for _, r := range p.rules {
-		if r.Disabled {
-			continue
-		}
-		thr := r.Threshold
-		out = append(out, api.AlertRuleInfo{Name: r.Name, Level: r.Level(), Op: r.Op, Threshold: &thr, Severity: r.Severity})
-	}
-	return append(out,
-		api.AlertRuleInfo{Name: alertpolicy.RuleDeviceDown, Level: alertpolicy.LevelDevice, Severity: "critical"},
-		api.AlertRuleInfo{Name: alertpolicy.RuleDeviceWarning, Level: alertpolicy.LevelDevice, Severity: "warning"},
-		api.AlertRuleInfo{Name: alertpolicy.RuleSystem, Level: alertpolicy.LevelSystem, Severity: "warning"})
-}
+// Rules implements api.AlertPolicyHost: every rule a policy can bind, from the registry.
+func (p *policyEngine) Rules() []api.AlertRuleInfo { return p.rules.Rules() }
 
 func (p *policyEngine) level(rule string) (string, bool) {
 	for _, r := range p.Rules() {
@@ -63,7 +50,7 @@ func (p *policyEngine) Preview(ctx context.Context, rule string, scope alertpoli
 	if err != nil {
 		return out, err
 	}
-	devs, err := p.activeDevices(ctx)
+	devs, err := activeDevices(ctx, p.st)
 	if err != nil {
 		return out, err
 	}
@@ -78,24 +65,6 @@ func (p *policyEngine) Preview(ctx context.Context, rule string, scope alertpoli
 	return out, nil
 }
 
-func (p *policyEngine) activeDevices(ctx context.Context) ([]store.Device, error) {
-	var all []store.Device
-	for off := int64(0); ; off += store.MaxLimit {
-		page, total, err := p.st.ListDevices(ctx, store.DeviceFilter{}, store.Page{Offset: off, Limit: store.MaxLimit})
-		if err != nil {
-			return nil, err
-		}
-		for _, d := range page {
-			if d.RetiredAt == nil && d.PurgedAt == nil && d.Enabled {
-				all = append(all, d)
-			}
-		}
-		if len(page) == 0 || off+int64(len(page)) >= total {
-			return all, nil
-		}
-	}
-}
-
 // previewOf evaluates one scope over devices and ports.
 type previewOf struct {
 	m       *alertpolicy.Matcher

internal/app/reload.go

@@ -35,6 +35,7 @@ const (
 	secSites     = "sites"
 	secProxmox   = "proxmox"
 	secChecks    = "checks"
+	secPoller    = "poller"
 )
 
 // liveConfig is the configuration as the running process sees it. running is what the restart-only parts were
@@ -99,6 +100,34 @@ func (a *App) Apply(ctx context.Context, actor, text string) (api.SettingsResult
 	if err != nil {
 		return api.SettingsResult{}, fmt.Errorf("%w: %w", api.ErrInvalidSettings, err)
 	}
+	a.settingsMu.Lock()
+	defer a.settingsMu.Unlock()
+	return a.applyConfig(ctx, actor, cfg)
+}
+
+// CurrentConfig implements api.SettingsConfigAdmin.
+func (a *App) CurrentConfig() (config.Config, []string) {
+	cur := a.live.get()
+	a.live.mu.Lock()
+	running := a.live.running
+	a.live.mu.Unlock()
+	return cur, restartSections(running, cur)
+}
+
+// UpdateConfig implements api.SettingsConfigAdmin: the typed form edits a copy of the current configuration, and
+// the result goes through the same validate, save and reload as a YAML document.
+func (a *App) UpdateConfig(ctx context.Context, actor string, mutate func(*config.Config) error) (api.SettingsResult, error) {
+	a.settingsMu.Lock()
+	defer a.settingsMu.Unlock()
+	cfg := a.live.get()
+	if err := mutate(&cfg); err != nil {
+		return api.SettingsResult{}, fmt.Errorf("%w: %w", api.ErrInvalidSettings, err)
+	}
+	return a.applyConfig(ctx, actor, cfg)
+}
+
+// applyConfig validates cfg, saves it and reloads what can change in place. The caller holds settingsMu.
+func (a *App) applyConfig(ctx context.Context, actor string, cfg config.Config) (api.SettingsResult, error) {
 	if err := cfg.Validate(); err != nil {
 		return api.SettingsResult{}, fmt.Errorf("%w: %w", api.ErrInvalidSettings, err)
 	}
@@ -130,6 +159,8 @@ func decodeStrict(text string) (config.Config, error) {
 
 // ReloadFile rereads config.yaml (SIGHUP). An invalid file changes nothing and returns the error.
 func (a *App) ReloadFile() (api.SettingsResult, error) {
+	a.settingsMu.Lock()
+	defer a.settingsMu.Unlock()
 	cfg, err := config.Load(a.dataDir)
 	if err != nil {
 		return api.SettingsResult{}, err
@@ -161,7 +192,12 @@ func (a *App) reloadWith(cfg config.Config) api.SettingsResult {
 		return a.policy.Reload(netpolicy.Config{Allow: al.OutboundAllow})
 	})
 	apply(!reflect.DeepEqual(old.Auth.TrustedProxies, cfg.Auth.TrustedProxies), secProxies, func() error {
-		return a.auth.SetTrustedProxies(cfg.Auth.TrustedProxies)
+		return a.curAuth().SetTrustedProxies(cfg.Auth.TrustedProxies)
+	})
+	apply(old.Poller != cfg.Poller, secPoller, func() error {
+		a.cadence.Set(config.EffectivePollInterval(a.flags, cfg))
+		a.poller.Refresh() // loops replan against the new default
+		return nil
 	})
 	apply(old.Backup != cfg.Backup, secBackup, func() error { a.setBackupSchedule(cfg); return nil })
 	apply(old.Disk != cfg.Disk, secDisk, func() error {

internal/app/rollup.go

@@ -38,7 +38,7 @@ func (a *App) startRollup(s *settings) error {
 	origin := a.installTime(context.Background(), clk)
 	job, err := rollup.New(rollup.Options{
 		Query: q, Inventory: a.st, State: a.st, Writer: ls, Deleter: ls, Queue: a.hot, Paused: a.guard.PauseRollup, Clock: clk,
-		Logger: a.log, Origin: origin,
+		Logger: a.log, Origin: origin, PollInterval: s.PollInterval, Cadence: a.cadence,
 		Gauges: profileGauges(a.profiles), Alarms: ruleAlarms{a.alerting.sys, alert.RuleRollupStalled},
 		HotRetention: s.cfg.Retention.Hot.D(), FiveMinuteRetention: s.cfg.Retention.FiveMinute.D(), LongRetention: s.cfg.Retention.Long.D(),
 	})

internal/app/rulecatalogue.go

@@ -0,0 +1,113 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"sort"
+	"time"
+
+	"github.com/v10networks/nmon/internal/alert"
+	"github.com/v10networks/nmon/internal/alertrule"
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/profile"
+)
+
+// catalogueTTL is how long the catalogue, including the push metric names read from the hot store, is cached.
+const catalogueTTL = time.Minute
+
+// pushNamesQuery lists the distinct metric names push ingest wrote.
+const pushNamesQuery = `count({push="1"}) by (__name__)`
+
+// Catalogue implements api.AlertRuleHost.
+func (r *ruleRegistry) Catalogue(ctx context.Context) api.RuleCatalogue {
+	return api.RuleCatalogue{
+		Metrics: r.catalogue(ctx).List(), Fields: alertrule.Fields(), Ops: alertrule.Ops,
+		Window: api.RuleWindowLimits{DefaultSeconds: int64(alertrule.DefaultWindow.Seconds()), MaxSeconds: int64(alertrule.MaxWindow.Seconds())},
+	}
+}
+
+// catalogue returns the metrics a rule may use. While the hot store cannot be asked it keeps the last one, so a
+// store hiccup neither refuses saves nor disables rules.
+func (r *ruleRegistry) catalogue(ctx context.Context) alertrule.Catalogue {
+	c, err := r.cat.get(struct{}{}, func() (alertrule.Catalogue, error) {
+		ms := r.profileMetrics()
+		push, err := r.pushMetrics(ctx)
+		if err != nil {
+			return alertrule.Catalogue{}, err
+		}
+		return alertrule.NewCatalogue(append(ms, push...)), nil
+	})
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if err != nil {
+		r.log.Debug("metric catalogue: push names unavailable", "err", err)
+		if r.lastCat.Len() == 0 {
+			r.lastCat = alertrule.NewCatalogue(r.profileMetrics())
+		}
+		return r.lastCat
+	}
+	r.lastCat = c
+	return c
+}
+
+// profileMetrics lists every pollable metric of every loaded profile.
+func (r *ruleRegistry) profileMetrics() []alertrule.MetricInfo {
+	set := r.profiles()
+	if set == nil {
+		return nil
+	}
+	var out []alertrule.MetricInfo
+	for _, n := range set.Names() {
+		res, ok := set.Lookup(n)
+		if !ok {
+			continue
+		}
+		for _, m := range res.Metrics {
+			if m.Pollable() {
+				out = append(out, metricInfo(m))
+			}
+		}
+	}
+	return out
+}
+
+func metricInfo(m profile.Metric) alertrule.MetricInfo {
+	level := alertrule.LevelDevice
+	switch m.Kind() {
+	case profile.KindInterface:
+		level = alertrule.LevelInterface
+	case profile.KindTable:
+		level = alertrule.LevelComponent
+	}
+	return alertrule.MetricInfo{Name: m.Name, Label: m.Title(), Unit: m.Unit, Type: string(m.Type), Category: m.Category, Of: m.Of,
+		Enum: m.Enum, Level: level, Source: alertrule.SourceProfile, IntervalSeconds: int64(m.Interval.D().Seconds())}
+}
+
+// pushMetrics reads the metric names push clients wrote, from the hot store.
+func (r *ruleRegistry) pushMetrics(ctx context.Context) ([]alertrule.MetricInfo, error) {
+	if r.query == nil {
+		return nil, nil
+	}
+	raw, err := r.query.Query(ctx, alert.StoreHot, pushNamesQuery, r.clk.Now())
+	if err != nil {
+		return nil, err
+	}
+	var resp struct {
+		Data struct {
+			Result []struct {
+				Metric map[string]string `json:"metric"`
+			} `json:"result"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(raw, &resp); err != nil {
+		return nil, err
+	}
+	var out []alertrule.MetricInfo
+	for _, s := range resp.Data.Result {
+		if n := s.Metric["__name__"]; n != "" {
+			out = append(out, alertrule.MetricInfo{Name: n, Label: n, Type: "gauge", Level: alertrule.LevelDevice, Source: alertrule.SourcePush})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out, nil
+}

internal/app/rulecopy.go

@@ -0,0 +1,76 @@
+package app
+
+import (
+	"context"
+
+	"github.com/v10networks/nmon/internal/alert"
+	"github.com/v10networks/nmon/internal/alertrule"
+	"github.com/v10networks/nmon/internal/api"
+)
+
+// copyRecipe expresses a built-in in the builder. Only built-ins whose query is one catalogue metric over one
+// window can be: the others join, filter or union series, which the builder deliberately cannot.
+type copyRecipe func(b alert.Rule) alertrule.MetricSpec
+
+// copyRecipes holds the built-ins the builder can reproduce exactly. cpu_high compiles to the very same
+// expression; the other two compare the raw value where the built-in compared a boolean of it, which breaches
+// in the same cases.
+var copyRecipes = map[string]copyRecipe{
+	"cpu_high": func(b alert.Rule) alertrule.MetricSpec {
+		return alertrule.MetricSpec{Metric: "hrProcessorLoad", Window: "15m", Func: alertrule.FuncLast, Aggregate: alertrule.AggAvg,
+			Op: b.Op, Threshold: b.Threshold}
+	},
+	"ups_on_battery": func(alert.Rule) alertrule.MetricSpec {
+		return alertrule.MetricSpec{Metric: "upsOutputSource", Window: "15m", Func: alertrule.FuncLast, Op: alertrule.OpEQ, Threshold: 5}
+	},
+	"printer_supply_empty": func(alert.Rule) alertrule.MetricSpec {
+		return alertrule.MetricSpec{Metric: "prtMarkerSuppliesLevel", Window: "1h", Func: alertrule.FuncLast, Op: alertrule.OpEQ, Threshold: 0}
+	},
+}
+
+// notCopyable says why each other built-in is out of the builder's reach.
+var notCopyable = map[string]string{
+	"interface_down":     "it joins two metrics (oper and admin status)",
+	"interface_errors":   "it divides the error rate by the packet rate, estimating packets from octets when needed",
+	"memory_high":        "it combines two memory sources",
+	"disk_full":          "it filters storage rows by name and also predicts when they fill",
+	"ups_runtime_low":    "it merges two vendors' runtime metrics",
+	"printer_supply_low": "it divides a supply level by its capacity and ignores the sentinel values",
+	"temp_high":          "it merges several vendors' temperature metrics",
+	"device_down":        "reachability rules are not metric rules",
+	"device_warning":     "reachability rules are not metric rules",
+	"system":             "system alarms are not metric rules",
+}
+
+// Copy implements api.AlertRuleHost. The returned draft keeps the built-in's severity, hold time and texts.
+func (r *ruleRegistry) Copy(ctx context.Context, name string) (alertrule.Draft, string) {
+	b, ok := r.builtin[name]
+	if !ok {
+		if why := notCopyable[name]; why != "" {
+			return alertrule.Draft{}, why
+		}
+		return alertrule.Draft{}, "it is not a built-in rule"
+	}
+	rec, ok := copyRecipes[name]
+	if !ok {
+		if why := notCopyable[name]; why != "" {
+			return alertrule.Draft{}, why
+		}
+		return alertrule.Draft{}, "the builder cannot express its query"
+	}
+	cur := b
+	for _, s := range r.static {
+		if s.Name == name {
+			cur = s // keep a config patch of its severity, threshold and hold time
+		}
+	}
+	ms := rec(cur)
+	d := alertrule.Draft{Name: name + "_custom", Kind: alertrule.KindMetric, Severity: cur.Severity, For: cur.For,
+		Summary: cur.Summary, Description: cur.Description, Spec: alertrule.RuleSpec{Metric: &ms}}
+	if _, err := alertrule.Build(d, r.catalogue(ctx), r.every()); err != nil {
+		return alertrule.Draft{}, "this install cannot build it: " + err.Error()
+	}
+	return d, ""
+}
+
+var _ api.AlertRuleHost = (*ruleRegistry)(nil)

internal/app/ruleinfo.go

@@ -0,0 +1,115 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"time"
+
+	"github.com/v10networks/nmon/internal/alert"
+	"github.com/v10networks/nmon/internal/alertpolicy"
+	"github.com/v10networks/nmon/internal/alertrule"
+	"github.com/v10networks/nmon/internal/api"
+)
+
+// Rules implements api.AlertRuleHost: built-in and config rules, then custom rules by name, then the
+// reachability and system rules that only policies can bind.
+func (r *ruleRegistry) Rules() []api.AlertRuleInfo {
+	out := make([]api.AlertRuleInfo, 0, len(r.static)+8)
+	for _, s := range r.static {
+		out = append(out, r.staticInfo(s))
+	}
+	cs := r.customRules()
+	sort.Slice(cs, func(i, j int) bool { return cs[i].row.Name < cs[j].row.Name })
+	for _, c := range cs {
+		if !r.Reserved(c.row.Name) {
+			out = append(out, customInfo(c))
+		}
+	}
+	return append(out,
+		reachInfo(alertpolicy.RuleDeviceDown, alertpolicy.LevelDevice, alert.SevCritical),
+		reachInfo(alertpolicy.RuleDeviceWarning, alertpolicy.LevelDevice, alert.SevWarning),
+		reachInfo(alertpolicy.RuleSystem, alertpolicy.LevelSystem, alert.SevWarning))
+}
+
+func reachInfo(name, level, sev string) api.AlertRuleInfo {
+	return api.AlertRuleInfo{ID: name, Name: name, Kind: "builtin", Enabled: true, Level: level, Severity: sev}
+}
+
+func (r *ruleRegistry) staticInfo(s alert.Rule) api.AlertRuleInfo {
+	kind := "config"
+	if d, ok := r.builtin[s.Name]; ok && d.Expr == s.Expr {
+		kind = "builtin"
+	}
+	thr, f := s.Threshold, s.For.String()
+	_, copyable := copyRecipes[s.Name]
+	return api.AlertRuleInfo{ID: s.Name, Name: s.Name, Kind: kind, Enabled: !s.Disabled, Level: s.Level(), Op: s.Op, Threshold: &thr,
+		Severity: s.Severity, For: &f, Summary: s.Summary, Expr: s.Expr, Copyable: copyable && kind == "builtin"}
+}
+
+func customInfo(c customRule) api.AlertRuleInfo {
+	thr, f := c.rule.Threshold, c.rule.For.String()
+	info := api.AlertRuleInfo{ID: c.row.ID, Name: c.row.Name, Kind: c.row.Kind, Editable: true, Enabled: c.row.Enabled,
+		Level: c.rule.Level(), Op: c.rule.Op, Threshold: &thr, Severity: c.rule.Severity, For: &f,
+		Summary: c.row.Summary, Expr: c.rule.Expr}
+	if c.row.ID != "" {
+		sp := c.spec
+		info.Spec = &sp
+	}
+	return info
+}
+
+// Build implements api.AlertRuleHost: it validates a draft against the live catalogue.
+func (r *ruleRegistry) Build(ctx context.Context, d alertrule.Draft) (api.RuleBuilt, error) {
+	b, err := alertrule.Build(d, r.catalogue(ctx), r.every())
+	if err != nil {
+		return api.RuleBuilt{}, &api.RuleInvalid{Err: err}
+	}
+	return api.RuleBuilt{Expr: b.Expr, Level: b.Level}, nil
+}
+
+// maxProbe bounds one dry run.
+const maxProbe = 30 * time.Second
+
+// Test implements api.AlertRuleHost: compile a draft and run it once, ignoring alert policies.
+func (r *ruleRegistry) Test(ctx context.Context, d alertrule.Draft) (api.RuleTestResult, error) {
+	if d.Name == "" {
+		d.Name = "draft"
+	}
+	b, err := alertrule.Build(d, r.catalogue(ctx), r.every())
+	if err != nil {
+		return api.RuleTestResult{}, &api.RuleInvalid{Err: err}
+	}
+	rule := alert.Rule{Name: d.Name, Severity: d.Severity, Expr: b.Expr, Op: b.Op, Threshold: b.Threshold, For: d.For, Labels: b.Labels,
+		Summary: b.Summary, Description: b.Description, SkipUnreliable: b.SkipUnreliable}
+	if b.Inventory != nil {
+		rule.Source = r.inv.source(b.Inventory)
+	}
+	cctx, cancel := context.WithTimeout(ctx, maxProbe)
+	defer cancel()
+	rds, err := alert.Probe(cctx, r.query, alert.StoreHot, rule, r.clk.Now())
+	if err != nil {
+		return api.RuleTestResult{}, &api.RuleInvalid{Err: errors.New("the rule could not run: " + err.Error())}
+	}
+	return testResult(b, rds), nil
+}
+
+func testResult(b alertrule.Built, rds []alert.ProbeReading) api.RuleTestResult {
+	sort.SliceStable(rds, func(i, j int) bool {
+		if rds[i].Breach != rds[j].Breach {
+			return rds[i].Breach
+		}
+		return rds[i].Device < rds[j].Device
+	})
+	res := api.RuleTestResult{Expr: b.Expr, Level: b.Level, Entities: len(rds), Samples: []api.RuleTestSample{}}
+	for _, rd := range rds {
+		if rd.Breach {
+			res.Breaching++
+		}
+		if len(res.Samples) < 20 {
+			res.Samples = append(res.Samples, api.RuleTestSample{Device: rd.Device, Iface: rd.Iface, Component: rd.Component,
+				Value: rd.Value, Breach: rd.Breach})
+		}
+	}
+	return res
+}

internal/app/ruleinventory.go

@@ -0,0 +1,162 @@
+package app
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"github.com/v10networks/nmon/internal/alert"
+	"github.com/v10networks/nmon/internal/alertrule"
+	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// inventoryTTL is how long the device and port snapshot an inventory rule reads is reused.
+const inventoryTTL = 30 * time.Second
+
+// inventory is every active device with its ports and parent count.
+type inventory struct {
+	devices []inventoryDevice
+}
+
+type inventoryDevice struct {
+	dev     store.Device
+	ifaces  []store.Interface
+	parents int
+}
+
+// inventoryReader hands inventory rules their rows from one shared, TTL-cached read of the store, so any
+// number of inventory rules cost one scan of the inventory per half minute.
+type inventoryReader struct {
+	st   ruleStore
+	log  *slog.Logger
+	snap *ttlCache[struct{}, *inventory]
+}
+
+func newInventoryReader(st ruleStore, clk clock.Clock, log *slog.Logger) *inventoryReader {
+	return &inventoryReader{st: st, log: log, snap: newTTLCache[struct{}, *inventory](clk, inventoryTTL)}
+}
+
+func (r *inventoryReader) invalidate() { r.snap.invalidate() }
+
+func (r *inventoryReader) load(ctx context.Context) (*inventory, error) {
+	return r.snap.get(struct{}{}, func() (*inventory, error) { return r.read(ctx) })
+}
+
+// read scans the active devices (not retired, purged or disabled), their live ports and the parent links.
+func (r *inventoryReader) read(ctx context.Context) (*inventory, error) {
+	devs, err := activeDevices(ctx, r.st)
+	if err != nil {
+		return nil, err
+	}
+	ifs, err := r.st.ListAllInterfaces(ctx)
+	if err != nil {
+		return nil, err
+	}
+	links, err := r.st.ListAllParents(ctx, "")
+	if err != nil {
+		return nil, err
+	}
+	byDev := map[int64][]store.Interface{}
+	for _, i := range ifs {
+		byDev[i.DeviceID] = append(byDev[i.DeviceID], i)
+	}
+	parents := map[int64]int{}
+	for _, l := range links {
+		parents[l.DeviceID]++
+	}
+	inv := &inventory{devices: make([]inventoryDevice, len(devs))}
+	for k, d := range devs {
+		inv.devices[k] = inventoryDevice{dev: d, ifaces: byDev[d.ID], parents: parents[d.ID]}
+	}
+	return inv, nil
+}
+
+// activeDevices lists devices that are neither retired, purged nor disabled.
+func activeDevices(ctx context.Context, st store.Devices) ([]store.Device, error) {
+	var all []store.Device
+	for off := int64(0); ; off += store.MaxLimit {
+		page, total, err := st.ListDevices(ctx, store.DeviceFilter{}, store.Page{Offset: off, Limit: store.MaxLimit})
+		if err != nil {
+			return nil, err
+		}
+		for _, d := range page {
+			if d.RetiredAt == nil && d.PurgedAt == nil && d.Enabled {
+				all = append(all, d)
+			}
+		}
+		if len(page) == 0 || off+int64(len(page)) >= total {
+			return all, nil
+		}
+	}
+}
+
+// inventorySource is one inventory rule's alert.Source.
+type inventorySource struct {
+	r    *inventoryReader
+	rule *alertrule.InventoryRule
+}
+
+func (r *inventoryReader) source(rule *alertrule.InventoryRule) alert.Source {
+	return inventorySource{r: r, rule: rule}
+}
+
+// Read reports 1 for every device or port whose conditions hold and 0 for the rest, so the state machine sees
+// "below" and resolves, never "no data".
+func (s inventorySource) Read(ctx context.Context, now time.Time) ([]alert.Reading, error) {
+	inv, err := s.r.load(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var out []alert.Reading
+	for _, d := range inv.devices {
+		if s.rule.Entity == alertrule.EntityDevice {
+			out = append(out, alert.Reading{Device: d.dev.Slug, Value: verdict(s.rule.Match(deviceRow(d, now)))})
+			continue
+		}
+		for _, i := range d.ifaces {
+			out = append(out, alert.Reading{Device: d.dev.Slug, Iface: i.ID, Value: verdict(s.rule.Match(ifaceRow(i, now)))})
+		}
+	}
+	return out, nil
+}
+
+func verdict(ok bool) float64 {
+	if ok {
+		return 1
+	}
+	return 0
+}
+
+// deviceRow maps a device to the whitelisted fields. A device never seen counts as unseen since it was created.
+func deviceRow(d inventoryDevice, now time.Time) alertrule.Row {
+	seen := d.dev.CreatedAt
+	if d.dev.LastSeen != nil {
+		seen = *d.dev.LastSeen
+	}
+	return alertrule.Row{
+		Num: map[string]float64{"parent_count": float64(d.parents)},
+		Bool: map[string]bool{
+			"profile_pinned": d.dev.ProfilePinned,
+			"snmp_failing":   d.dev.PollState == store.PollDegraded || d.dev.PollState == store.PollBackoff,
+		},
+		Str: map[string]string{"profile": d.dev.Profile, "site": d.dev.Site, "role": d.dev.Role, "kind": d.dev.Kind,
+			"sys_descr": d.dev.SysDescr, "hw_serial": d.dev.HWSerial, "mgmt_ip": d.dev.MgmtIP},
+		Dur: map[string]time.Duration{"last_seen_age": max(0, now.Sub(seen))},
+	}
+}
+
+// ifaceRow maps a port to the whitelisted fields. Speed is in Mb/s, from ifHighSpeed or else ifSpeed.
+func ifaceRow(i store.Interface, now time.Time) alertrule.Row {
+	speed := i.IfHighSpeed
+	if speed == 0 {
+		speed = i.IfSpeed / 1_000_000
+	}
+	return alertrule.Row{
+		Num: map[string]float64{"if_speed_mbps": float64(speed), "admin_status": float64(i.AdminStatus),
+			"oper_status": float64(i.OperStatus), "counter_discontinuity": float64(i.CounterDiscontinuity)},
+		Bool: map[string]bool{"unreliable": i.Unreliable, "has_hc_counters": i.HasHCCounters, "excluded": i.Excluded},
+		Str:  map[string]string{"if_name": i.IfName, "if_alias": i.IfAlias, "description": i.Description},
+		Dur:  map[string]time.Duration{"last_discovered_age": max(0, now.Sub(i.LastDiscovered))},
+	}
+}

internal/app/rules.go

@@ -0,0 +1,210 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/v10networks/nmon/internal/alert"
+	"github.com/v10networks/nmon/internal/alertpolicy"
+	"github.com/v10networks/nmon/internal/alertrule"
+	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/profile"
+	"github.com/v10networks/nmon/internal/store"
+)
+
+// ruleRefresh is how often the custom rule table is re-read; an API write invalidates it at once.
+const ruleRefresh = 30 * time.Second
+
+// ruleStore is what the rule registry reads.
+type ruleStore interface {
+	store.AlertRules
+	store.Devices
+	store.InterfaceInventory
+	store.ParentLinks
+}
+
+// ruleRegistry is the one list of alert rules: the built-ins, then config.yaml rules (deprecated), then the
+// custom rules in the database. The evaluator asks it for the rule set every cycle, so an edit applies on the
+// next cycle without a restart. It also backs the API: the catalogue the builder offers, validation, and a dry run.
+type ruleRegistry struct {
+	st       ruleStore
+	clk      clock.Clock
+	log      *slog.Logger
+	static   []alert.Rule // built-ins with their config patches, then config rules
+	builtin  map[string]alert.Rule
+	pollEach time.Duration
+	cadence  *store.PollCadence // the live default; overrides pollEach when set
+	query    alert.Querier
+	profiles func() *profile.Set
+	inv      *inventoryReader
+
+	custom  *ttlCache[struct{}, []customRule]
+	cat     *ttlCache[struct{}, alertrule.Catalogue]
+	mu      sync.Mutex
+	lastCat alertrule.Catalogue
+	lastOK  map[string]alert.Rule // last rule that compiled, by name, kept while the catalogue is briefly short
+}
+
+// customRule is a stored rule and what it compiled to.
+type customRule struct {
+	row   store.AlertRule
+	spec  alertrule.RuleSpec
+	built alertrule.Built
+	rule  alert.Rule
+	err   error
+}
+
+func newRuleRegistry(st ruleStore, clk clock.Clock, static []alert.Rule, pollEach time.Duration, q alert.Querier,
+	profiles func() *profile.Set, log *slog.Logger) *ruleRegistry {
+	r := &ruleRegistry{st: st, clk: clk, log: log, static: static, builtin: map[string]alert.Rule{}, pollEach: pollEach,
+		query: q, profiles: profiles, lastOK: map[string]alert.Rule{}}
+	for _, d := range alert.DefaultRules() {
+		r.builtin[d.Name] = d
+	}
+	r.inv = newInventoryReader(st, clk, log)
+	r.custom = newTTLCache[struct{}, []customRule](clk, ruleRefresh)
+	r.cat = newTTLCache[struct{}, alertrule.Catalogue](clk, catalogueTTL)
+	return r
+}
+
+// Changed drops the caches, so a rule write applies on the next evaluation cycle.
+func (r *ruleRegistry) Changed() {
+	r.custom.invalidate()
+	r.inv.invalidate()
+}
+
+// Reserved reports a name a custom rule cannot take.
+func (r *ruleRegistry) Reserved(name string) bool {
+	for _, s := range r.static {
+		if s.Name == name {
+			return true
+		}
+	}
+	switch name {
+	case alertpolicy.RuleDeviceDown, alertpolicy.RuleDeviceWarning, alertpolicy.RuleSystem:
+		return true
+	}
+	return alert.SystemRule(name)
+}
+
+// Set is alert.Options.RuleSet: the static rules plus every custom rule that compiles. A custom rule that does
+// not compile now (its metric left the catalogue for a moment) keeps its last good form.
+func (r *ruleRegistry) Set() []alert.Rule {
+	out := append([]alert.Rule(nil), r.static...)
+	for _, c := range r.customRules() {
+		if r.Reserved(c.row.Name) {
+			r.log.Warn("custom alert rule ignored: its name is taken by a built-in or config rule", "rule", c.row.Name)
+			continue
+		}
+		out = append(out, c.rule)
+	}
+	return out
+}
+
+// customRules returns the stored rules, compiled. While the store fails it returns the last good list.
+func (r *ruleRegistry) customRules() []customRule {
+	cs, err := r.custom.get(struct{}{}, func() ([]customRule, error) { return r.loadCustom(context.Background()) })
+	if err != nil {
+		r.log.Warn("load custom alert rules", "err", err)
+		r.mu.Lock()
+		defer r.mu.Unlock()
+		for _, old := range r.lastOK {
+			if !r.isStatic(old.Name) {
+				cs = append(cs, customRule{row: store.AlertRule{Name: old.Name}, rule: old})
+			}
+		}
+	}
+	return cs
+}
+
+func (r *ruleRegistry) isStatic(name string) bool {
+	for _, s := range r.static {
+		if s.Name == name {
+			return true
+		}
+	}
+	return false
+}
+
+func (r *ruleRegistry) loadCustom(ctx context.Context) ([]customRule, error) {
+	rows, err := r.st.ListAlertRules(ctx)
+	if err != nil {
+		return nil, err
+	}
+	cat := r.catalogue(ctx)
+	out := make([]customRule, 0, len(rows))
+	live := map[string]bool{}
+	for _, row := range rows {
+		c := r.compileRow(row, cat)
+		live[row.Name] = true
+		r.remember(c)
+		if c.err != nil {
+			r.log.Warn("custom alert rule does not compile", "rule", row.Name, "err", c.err)
+			if old, ok := r.lastGood(row.Name); ok {
+				c.rule = old
+				c.rule.Disabled = !row.Enabled
+			} else {
+				continue
+			}
+		}
+		out = append(out, c)
+	}
+	r.forget(live)
+	return out, nil
+}
+
+func (r *ruleRegistry) remember(c customRule) {
+	if c.err != nil {
+		return
+	}
+	r.mu.Lock()
+	r.lastOK[c.row.Name] = c.rule
+	r.mu.Unlock()
+}
+
+func (r *ruleRegistry) lastGood(name string) (alert.Rule, bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	rule, ok := r.lastOK[name]
+	return rule, ok
+}
+
+// forget drops last-good entries of rules that no longer exist.
+func (r *ruleRegistry) forget(live map[string]bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for n := range r.lastOK {
+		if !live[n] {
+			delete(r.lastOK, n)
+		}
+	}
+}
+
+// compileRow turns a stored rule into an evaluator rule.
+func (r *ruleRegistry) compileRow(row store.AlertRule, cat alertrule.Catalogue) customRule {
+	c := customRule{row: row}
+	if err := json.Unmarshal([]byte(row.Spec), &c.spec); err != nil {
+		c.err = fmt.Errorf("spec: %w", err)
+		return c
+	}
+	d := alertrule.Draft{Name: row.Name, Kind: row.Kind, Severity: row.Severity, For: time.Duration(row.ForSeconds) * time.Second,
+		Summary: row.Summary, Description: row.Description, Spec: c.spec}
+	c.built, c.err = alertrule.Build(d, cat, r.every())
+	if c.err != nil {
+		return c
+	}
+	c.rule = alert.Rule{Name: row.Name, Severity: row.Severity, Expr: c.built.Expr, Op: c.built.Op, Threshold: c.built.Threshold,
+		For: d.For, Labels: c.built.Labels, Summary: c.built.Summary, Description: c.built.Description,
+		SkipUnreliable: c.built.SkipUnreliable, Disabled: !row.Enabled}
+	if c.built.Inventory != nil {
+		c.rule.Source = r.inv.source(c.built.Inventory)
+	}
+	return c
+}
+
+// every is the default poll interval rules are built against: the live cadence, else the fixed one.
+func (r *ruleRegistry) every() time.Duration { return r.cadence.Or(r.pollEach) }

internal/app/rules_test.go

@@ -0,0 +1,337 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/v10networks/nmon/internal/alert"
+	"github.com/v10networks/nmon/internal/alertpolicy"
+	"github.com/v10networks/nmon/internal/alertrule"
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/profile"
+	"github.com/v10networks/nmon/internal/selfmon"
+	"github.com/v10networks/nmon/internal/store"
+	"github.com/v10networks/nmon/profiles"
+)
+
+// ruleQuerier answers instant queries from a table of expr -> device -> value, and the push name query.
+type ruleQuerier struct {
+	mu     sync.Mutex
+	series map[string]map[string]float64
+	push   []string
+}
+
+func (q *ruleQuerier) set(expr, device string, v float64) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	if q.series == nil {
+		q.series = map[string]map[string]float64{}
+	}
+	if q.series[expr] == nil {
+		q.series[expr] = map[string]float64{}
+	}
+	q.series[expr][device] = v
+}
+
+func (q *ruleQuerier) Query(_ context.Context, _, expr string, _ time.Time) ([]byte, error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	var res []map[string]any
+	if expr == pushNamesQuery {
+		for _, n := range q.push {
+			res = append(res, map[string]any{"metric": map[string]string{"__name__": n}, "value": []any{1, "1"}})
+		}
+	}
+	for dev, v := range q.series[expr] {
+		res = append(res, map[string]any{"metric": map[string]string{"device": dev}, "value": []any{1, jsonNum(v)}})
+	}
+	return json.Marshal(map[string]any{"status": "success", "data": map[string]any{"resultType": "vector", "result": res}})
+}
+
+func jsonNum(v float64) string { b, _ := json.Marshal(v); return string(b) }
+
+type ruleNotifier struct {
+	mu     sync.Mutex
+	states []string
+}
+
+func (n *ruleNotifier) Notify(_ context.Context, t alert.Transition) error {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	n.states = append(n.states, t.Rule+":"+t.State)
+	return nil
+}
+
+func (n *ruleNotifier) got() string {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	return strings.Join(n.states, ",")
+}
+
+type ruleEnv struct {
+	*policyEnv
+	q   *ruleQuerier
+	n   *ruleNotifier
+	ev  *alert.Evaluator
+	set *profile.Set
+}
+
+func newRuleEnv(t *testing.T) *ruleEnv {
+	t.Helper()
+	e := &ruleEnv{policyEnv: newPolicyEnv(t), q: &ruleQuerier{}, n: &ruleNotifier{}}
+	var err error
+	if e.set, err = profile.LoadFS(profiles.FS); err != nil {
+		t.Fatal(err)
+	}
+	e.reg = newRuleRegistry(e.st, e.clk, alert.DefaultRules(), time.Minute, e.q, func() *profile.Set { return e.set }, testLogger())
+	e.pe = newPolicyEngine(e.st, e.clk, newOptInSet(e.st, e.clk, entityTTL, testLogger()), e.reg, testLogger())
+	e.ev, err = alert.New(alert.Options{Store: e.st, Querier: e.q, Notifier: e.n, Clock: e.clk, RuleSet: e.reg.Set,
+		Policy: e.pe.Metric, Metrics: selfmon.New(), Logger: testLogger()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return e
+}
+
+func (e *ruleEnv) addRule(name, kind string, spec alertrule.RuleSpec) *store.AlertRule {
+	e.t.Helper()
+	js, err := json.Marshal(spec)
+	if err != nil {
+		e.t.Fatal(err)
+	}
+	r := &store.AlertRule{Name: name, Kind: kind, Enabled: true, Severity: "warning", Spec: string(js)}
+	if err := e.st.CreateAlertRule(context.Background(), r); err != nil {
+		e.t.Fatal(err)
+	}
+	e.reg.Changed()
+	return r
+}
+
+func (e *ruleEnv) cycle() {
+	e.clk.Advance(30 * time.Second)
+	e.ev.EvalOnce(context.Background())
+}
+
+func cpuSpec(threshold float64) alertrule.RuleSpec {
+	return alertrule.RuleSpec{Metric: &alertrule.MetricSpec{Metric: "hrProcessorLoad", Aggregate: alertrule.AggMax, Op: ">", Threshold: threshold}}
+}
+
+func (e *ruleEnv) exprOf(rule string) string {
+	e.t.Helper()
+	for _, r := range e.reg.Set() {
+		if r.Name == rule {
+			return r.Expr
+		}
+	}
+	e.t.Fatalf("rule %s not in the rule set", rule)
+	return ""
+}
+
+func TestCustomMetricRuleBoundByAPolicyFiresResolvesAndEditsApplyLive(t *testing.T) {
+	e := newRuleEnv(t)
+	row := e.addRule("cpu_custom", alertrule.KindMetric, cpuSpec(80))
+	e.policy("cpu everywhere", "cpu_custom", alertpolicy.ActionAlert, 0, alertpolicy.Scope{All: true})
+	expr := e.exprOf("cpu_custom")
+	if expr != `max by (device) (last_over_time(hrProcessorLoad[10m]))` {
+		t.Fatalf("expr = %q", expr)
+	}
+	e.q.set(expr, e.dev.Slug, 95)
+	e.cycle()
+	if got := e.n.got(); got != "cpu_custom:firing" {
+		t.Fatalf("after breach: %q", got)
+	}
+	e.q.set(expr, e.dev.Slug, 50)
+	e.cycle()
+	if got := e.n.got(); got != "cpu_custom:firing,cpu_custom:resolved" {
+		t.Fatalf("after recovery: %q", got)
+	}
+	// An edit (threshold 40) applies on the very next cycle, no restart: 50 now breaches.
+	js, _ := json.Marshal(cpuSpec(40))
+	row.Spec = string(js)
+	if err := e.st.UpdateAlertRule(context.Background(), row); err != nil {
+		t.Fatal(err)
+	}
+	e.reg.Changed()
+	e.cycle()
+	if got := e.n.got(); !strings.HasSuffix(got, "cpu_custom:resolved,cpu_custom:firing") {
+		t.Fatalf("after edit: %q", got)
+	}
+	// Disabling retires the open alert silently: no resolved notification.
+	row.Enabled = false
+	if err := e.st.UpdateAlertRule(context.Background(), row); err != nil {
+		t.Fatal(err)
+	}
+	e.reg.Changed()
+	before := e.n.got()
+	e.cycle()
+	if e.n.got() != before {
+		t.Fatalf("disable notified: %q", e.n.got())
+	}
+	open, err := e.st.ListActiveAlertStates(context.Background())
+	if err != nil || len(open) != 0 {
+		t.Fatalf("open alerts after disable: %v %v", open, err)
+	}
+}
+
+func TestInventoryRuleFiresForTheMatchingPortOnly(t *testing.T) {
+	e := newRuleEnv(t)
+	id := func(idx int64, name, alias string, mbps int64) string {
+		i := e.iface(idx, name, alias, 6)
+		got, err := e.st.GetInterface(context.Background(), i)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got.IfHighSpeed = mbps
+		if err := e.st.UpsertInterfaces(context.Background(), e.dev.ID, []store.Interface{*got}); err != nil {
+			t.Fatal(err)
+		}
+		return i
+	}
+	bare := id(1, "uplink1", "", 10000)
+	_ = id(2, "uplink2", "to core", 10000)
+	_ = id(3, "access1", "", 1000)
+	e.addRule("uplink_no_desc", alertrule.KindInventory, alertrule.RuleSpec{Inventory: &alertrule.InventorySpec{Conditions: []alertrule.InventoryCondition{
+		{Field: "if_alias", Op: "empty"}, {Field: "if_speed_mbps", Op: ">=", Value: "10000"}}}})
+	e.policy("all ports", "uplink_no_desc", alertpolicy.ActionAlert, 0, alertpolicy.Scope{All: true})
+	e.cycle()
+	as, _, err := e.st.ListAlerts(context.Background(), store.AlertFilter{}, store.Page{Limit: 100})
+	if err != nil || len(as) != 1 || as[0].Iface != bare || as[0].State != store.AlertFiring {
+		t.Fatalf("alerts: %+v %v", as, err)
+	}
+	if info := findInfo(e.reg.Rules(), "uplink_no_desc"); info.Level != "interface" || info.Kind != "inventory" || !info.Editable {
+		t.Fatalf("info: %+v", info)
+	}
+}
+
+func findInfo(infos []api.AlertRuleInfo, name string) api.AlertRuleInfo {
+	for _, i := range infos {
+		if i.Name == name {
+			return i
+		}
+	}
+	return api.AlertRuleInfo{}
+}
+
+func TestCustomRuleNamesCannotShadowBuiltinsOrConfig(t *testing.T) {
+	e := newRuleEnv(t)
+	for _, n := range []string{"cpu_high", "device_down", "device_warning", "system", "interface_errors"} {
+		if !e.reg.Reserved(n) {
+			t.Errorf("%s is not reserved", n)
+		}
+	}
+	if e.reg.Reserved("my_rule") {
+		t.Error("my_rule reserved")
+	}
+	// A stored row that collides is ignored, so the built-in keeps its meaning.
+	e.addRule("cpu_high", alertrule.KindMetric, cpuSpec(1))
+	n := 0
+	for _, r := range e.reg.Set() {
+		if r.Name == "cpu_high" {
+			n++
+			if r.Threshold != 90 {
+				t.Fatalf("built-in overridden: %+v", r)
+			}
+		}
+	}
+	if n != 1 {
+		t.Fatalf("cpu_high appears %d times", n)
+	}
+}
+
+func TestCopyBuiltinsToTheBuilder(t *testing.T) {
+	e := newRuleEnv(t)
+	ctx := context.Background()
+	d, why := e.reg.Copy(ctx, "cpu_high")
+	if why != "" {
+		t.Fatal(why)
+	}
+	b, err := alertrule.Build(d, e.reg.catalogue(ctx), time.Minute)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := alert.DefaultRules()[2].Expr; b.Expr != want {
+		t.Fatalf("copy compiles to %q, built-in is %q", b.Expr, want)
+	}
+	for _, name := range []string{"ups_on_battery", "printer_supply_empty"} {
+		if _, why := e.reg.Copy(ctx, name); why != "" {
+			t.Errorf("%s: %s", name, why)
+		}
+	}
+	for _, name := range []string{"interface_down", "interface_errors", "memory_high", "disk_full", "ups_runtime_low", "printer_supply_low",
+		"temp_high", "device_down", "system"} {
+		if _, why := e.reg.Copy(ctx, name); why == "" {
+			t.Errorf("%s was copyable", name)
+		}
+	}
+	if _, why := e.reg.Copy(ctx, "no_such"); why == "" {
+		t.Error("unknown rule copyable")
+	}
+}
+
+func TestCatalogueHasProfileAndPushMetrics(t *testing.T) {
+	e := newRuleEnv(t)
+	e.q.push = []string{"room_temp", `bad"name`}
+	e.reg.cat.invalidate()
+	cat := e.reg.catalogue(context.Background())
+	for _, n := range []string{"hrProcessorLoad", "ifHCInOctets", "ifOperStatus", "room_temp", "prtMarkerSuppliesLevel"} {
+		if _, ok := cat.Lookup(n); !ok {
+			t.Errorf("%s missing from the catalogue", n)
+		}
+	}
+	if m, _ := cat.Lookup("room_temp"); m.Source != alertrule.SourcePush {
+		t.Errorf("room_temp source = %q", m.Source)
+	}
+	if _, ok := cat.Lookup(`bad"name`); ok {
+		t.Error("a name that is not an identifier entered the catalogue")
+	}
+	// Port speed is not a stored series, so utilisation is not offered.
+	for _, m := range cat.List() {
+		for _, tr := range m.Transforms {
+			if tr == "utilisation" {
+				t.Fatalf("%s offers utilisation", m.Name)
+			}
+		}
+	}
+	if _, ok := cat.Lookup("ifHighSpeed"); ok {
+		t.Error("ifHighSpeed is polled as a series after all; revisit utilisation")
+	}
+}
+
+func TestTestRunReportsBreachingEntities(t *testing.T) {
+	e := newRuleEnv(t)
+	ms := cpuSpec(80)
+	d := alertrule.Draft{Name: "draft", Kind: alertrule.KindMetric, Severity: "warning", Spec: ms}
+	expr := `max by (device) (last_over_time(hrProcessorLoad[10m]))`
+	e.q.set(expr, "a", 95)
+	e.q.set(expr, "b", 10)
+	res, err := e.reg.Test(context.Background(), d)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.Expr != expr || res.Entities != 2 || res.Breaching != 1 || res.Samples[0].Device != "a" || !res.Samples[0].Breach {
+		t.Fatalf("result: %+v", res)
+	}
+	d.Spec.Metric.Metric = "nope"
+	if _, err := e.reg.Test(context.Background(), d); err == nil {
+		t.Fatal("unknown metric tested")
+	}
+}
+
+// TestTemplateDataMirrorsTheEvaluators keeps alertrule's save-time template check in step with the data the
+// evaluator renders with.
+func TestTemplateDataMirrorsTheEvaluators(t *testing.T) {
+	a, b := reflect.TypeOf(alert.TemplateData{}), reflect.TypeOf(alertrule.TemplateData{})
+	if a.NumField() != b.NumField() {
+		t.Fatalf("%d fields vs %d", a.NumField(), b.NumField())
+	}
+	for i := 0; i < a.NumField(); i++ {
+		if a.Field(i).Name != b.Field(i).Name || a.Field(i).Type != b.Field(i).Type {
+			t.Errorf("field %d: %v vs %v", i, a.Field(i), b.Field(i))
+		}
+	}
+}

internal/app/security.go

@@ -3,10 +3,14 @@ package app
 import (
 	"crypto/tls"
 	"errors"
+	"fmt"
 	"log/slog"
+	"net"
 	"net/http"
+	"os"
 
 	"github.com/v10networks/nmon/internal/auth"
+	"github.com/v10networks/nmon/internal/clock"
 	"github.com/v10networks/nmon/internal/config"
 )
 
@@ -19,20 +23,55 @@ type transport struct {
 	files    *auth.CertFiles // provided certificate, reloaded on SIGHUP; nil with ACME or plain HTTP
 	cfg      *tls.Config     // nil for plain HTTP
 	acme     http.Handler    // the http-01 challenge handler when ACME is on
+
+	// httpAddr is the --http-listen address for a reverse proxy; empty when the listener is off. Unlike
+	// insecure, it never turns authentication off.
+	httpAddr string
+	// selfSigned: no certificate was given, so nmon generated (or reused) one under <data-dir>/tls.
+	selfSigned bool
+	// httpOnly: no TLS is configured, so httpAddr is the only listener and --listen is not bound.
+	httpOnly bool
 }
 
 // planTransport decides how to serve and refuses the combinations that would expose nmon unprotected:
 // no TLS without --insecure-http, both at once, or half a certificate pair.
 func planTransport(f *config.Flags, dataDir string) (*transport, error) {
+	if f.HTTPListen != "" && f.InsecureHTTP {
+		return nil, errors.New("--http-listen (plain HTTP behind a reverse proxy, sign-in required) and --insecure-http " +
+			"(development mode, no sign-in) are mutually exclusive; use one")
+	}
+	tr, err := planTLS(f, dataDir)
+	if err != nil {
+		return nil, err
+	}
+	tr.httpAddr = f.HTTPListen
+	return tr, nil
+}
+
+// checkTLSFlags refuses half a certificate pair and flag combinations that contradict each other.
+func checkTLSFlags(f *config.Flags) error {
 	provided := f.TLSCert != "" || f.TLSKey != ""
 	acme := len(f.ACMEHosts) > 0
 	switch {
 	case provided && (f.TLSCert == "" || f.TLSKey == ""):
-		return nil, errors.New("--tls-cert and --tls-key go together")
+		return errors.New("--tls-cert and --tls-key go together")
 	case provided && acme:
-		return nil, errors.New("use either --tls-cert/--tls-key or --acme-host, not both")
+		return errors.New("use either --tls-cert/--tls-key or --acme-host, not both")
 	case f.InsecureHTTP && (provided || acme):
-		return nil, errors.New("--insecure-http serves plain HTTP; it cannot be combined with TLS flags")
+		return errors.New("--insecure-http serves plain HTTP; it cannot be combined with TLS flags")
+	}
+	return nil
+}
+
+// planTLS picks the primary listener's transport from the TLS flags; --http-listen alone yields a plain-HTTP-only
+// transport instead of the "no certificate" refusal.
+func planTLS(f *config.Flags, dataDir string) (*transport, error) {
+	if err := checkTLSFlags(f); err != nil {
+		return nil, err
+	}
+	provided := f.TLSCert != "" || f.TLSKey != ""
+	acme := len(f.ACMEHosts) > 0
+	switch {
 	case f.InsecureHTTP:
 		return &transport{insecure: true}, nil
 	case provided:
@@ -47,32 +86,64 @@ func planTransport(f *config.Flags, dataDir string) (*transport, error) {
 			return nil, err
 		}
 		return &transport{cfg: cfg, acme: challenge}, nil
+	case f.HTTPListen != "":
+		return &transport{httpOnly: true}, nil
 	}
-	return nil, errors.New("nmon serves HTTPS by default and has no certificate: pass --tls-cert and --tls-key, " +
-		"or --acme-host (repeatable) and --acme-email for Let's Encrypt; " +
-		"for development only, --insecure-http serves plain HTTP")
+	return selfSignedTransport(dataDir)
+}
+
+// selfSignedTransport is the fresh-install default: HTTPS with a generated certificate, so nmon never starts on
+// plain HTTP by accident. The pair is created once and reused; replace it with --tls-cert/--tls-key or --acme-host.
+func selfSignedTransport(dataDir string) (*transport, error) {
+	host, _ := os.Hostname()
+	addrs, _ := net.InterfaceAddrs() // a failure leaves only the loopback names, which is still a working certificate
+	certFile, keyFile, _, err := auth.EnsureSelfSigned(dataDir, auth.LocalNames(host, addrs), clock.Real())
+	if err != nil {
+		return nil, fmt.Errorf("self-signed certificate: %w", err)
+	}
+	files, err := auth.LoadCertFiles(certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+	return &transport{files: files, cfg: files.Config(), selfSigned: true}, nil
+}
+
+// warnSelfSigned tells the operator which certificate browsers will complain about and how to replace it.
+func warnSelfSigned(log *slog.Logger, tr *transport, dataDir string) {
+	if !tr.selfSigned {
+		return
+	}
+	log.Warn("a SELF-SIGNED certificate is in use, so browsers will warn; replace it with --tls-cert and --tls-key "+
+		"(reloaded on SIGHUP) or --acme-host for Let's Encrypt", "dir", auth.SelfSignedDir(dataDir))
 }
 
 // authMode says how callers are authenticated.
 type authMode struct {
 	// devOpen: no authentication; only allowed together with --insecure-http.
 	devOpen bool
+	// install: first-install setup is open to whoever can reach nmon (see install.go).
+	install bool
 }
 
-// planAuth refuses to serve over TLS with no way to sign in, and turns authentication off only for
-// --insecure-http with no OIDC and no break-glass file (development mode, DEVIATIONS 31).
-func planAuth(tr *transport, a config.Auth) (authMode, error) {
-	configured := a.OIDC != nil || a.BreakGlassFile != ""
+// planAuth decides how callers are authenticated. A configured sign-in method means normal authentication;
+// --insecure-http with none is development mode (DEVIATIONS 31); otherwise nmon is in install mode until setup has
+// been completed, and refuses to serve once it has (an install that lost its sign-in must not reopen setup).
+func planAuth(tr *transport, a config.Auth, setupCompleted bool) (authMode, error) {
 	switch {
-	case configured:
+	case signInConfigured(a):
 		return authMode{}, nil
 	case tr.insecure:
 		return authMode{devOpen: true}, nil
+	case !setupCompleted:
+		return authMode{install: true}, nil
 	}
 	return authMode{}, errors.New("no way to sign in is configured: set auth.oidc and/or auth.break_glass_file in config.yaml " +
-		"(nmon will not serve over TLS with authentication off)")
+		"(nmon will not serve over TLS or --http-listen with authentication off, and setup was already completed)")
 }
 
+// signInConfigured reports whether config.yaml names OIDC or a break-glass file.
+func signInConfigured(a config.Auth) bool { return a.OIDC != nil || a.BreakGlassFile != "" }
+
 // warnDevOpen logs, loudly, that every request is an admin.
 func warnDevOpen(log *slog.Logger, addr string) {
 	log.Warn("AUTHENTICATION IS DISABLED (development mode): every request is the admin \"dev\"; "+
@@ -85,8 +156,28 @@ func (t *transport) describe() string {
 	switch {
 	case t.insecure:
 		return "plain HTTP (--insecure-http)"
-	case t.acme != nil:
-		return "HTTPS with ACME certificates"
+	case t.httpOnly:
+		return "plain HTTP only (--http-listen, for a reverse proxy)"
 	}
-	return "HTTPS with the provided certificate"
+	d := "HTTPS with the provided certificate"
+	if t.selfSigned {
+		d = "HTTPS with a self-signed certificate"
+	}
+	if t.acme != nil {
+		d = "HTTPS with ACME certificates"
+	}
+	if t.httpAddr != "" {
+		d += " and plain HTTP on " + t.httpAddr + " (--http-listen, for a reverse proxy)"
+	}
+	return d
+}
+
+// warnNoTrustedProxies tells the operator that without auth.trusted_proxies the proxy hides every client.
+func warnNoTrustedProxies(log *slog.Logger, tr *transport, a config.Auth) {
+	if tr.httpAddr == "" || len(a.TrustedProxies) > 0 {
+		return
+	}
+	log.Warn("--http-listen is on but auth.trusted_proxies is empty: X-Forwarded-For and X-Forwarded-Proto are ignored, so "+
+		"every client is seen as the proxy's address (break-glass lockout becomes per-proxy, audit shows the proxy) and "+
+		"session cookies are not marked Secure; list the proxy's address in auth.trusted_proxies", "http_listen", tr.httpAddr)
 }

internal/app/settingsform_test.go

@@ -0,0 +1,31 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/api"
+	"github.com/v10networks/nmon/internal/config"
+)
+
+func TestUpdateConfigEditsOneSectionThroughTheYAMLValidationPath(t *testing.T) {
+	a, f := startWith(t, nil, "")
+	ctx := context.Background()
+	if _, err := a.Apply(ctx, "tester", "timezone: America/Toronto\nbackup: {at: \"01:00\"}\n"); err != nil {
+		t.Fatal(err)
+	}
+	_, err := a.UpdateConfig(ctx, "tester", func(c *config.Config) error { c.Disk = config.Disk{AlertUsed: 0.95, AdmitUsed: 0.9}; return nil })
+	if !errors.Is(err, api.ErrInvalidSettings) {
+		t.Fatalf("an invalid section: %v, want ErrInvalidSettings", err)
+	}
+	res, err := a.UpdateConfig(ctx, "tester", func(c *config.Config) error { c.Disk = config.Disk{AlertUsed: 0.7}; return nil })
+	if err != nil || strings.Join(res.Applied, " ") != "disk" {
+		t.Fatalf("applied %v, err %v", res.Applied, err)
+	}
+	cfg, err := config.Load(f.DataDir)
+	if err != nil || cfg.Timezone != "America/Toronto" || cfg.Backup.At != "01:00" || cfg.Disk.AlertUsed != 0.7 {
+		t.Fatalf("saved config = %+v, err %v: the other sections must be untouched", cfg, err)
+	}
+}

internal/app/startup.go

@@ -71,11 +71,8 @@ func resolveSettings(f config.Flags) (*settings, error) {
 	if err != nil {
 		return nil, err
 	}
-	mode, err := planAuth(tr, cfg.Auth)
-	if err != nil {
-		return nil, err
-	}
-	s := &settings{Flags: f, keyFile: keyFile, driver: f.DBDriver, dsn: f.DBDSN, cfg: cfg, alerting: al, tr: tr, mode: mode}
+	// The auth mode needs the stored "setup completed" flag, so Start decides it once the inventory is open.
+	s := &settings{Flags: f, keyFile: keyFile, driver: f.DBDriver, dsn: f.DBDSN, cfg: cfg, alerting: al, tr: tr}
 	if s.driver == "" {
 		s.driver = store.DriverSQLite
 	}

internal/app/wiring.go

@@ -92,7 +92,8 @@ func (a *App) loadProfiles(s *settings) error {
 func (a *App) startPoller(s *settings) {
 	dir := s.profilesDir()
 	a.poller = poller.New(poller.Options{
-		Store: a.st, Writer: a.hotW, Interval: s.PollInterval, MaxInflight: s.MaxInflight, SNMPPort: s.SNMPPort, Logger: a.log,
+		Store: a.st, Writer: a.hotW, Interval: s.PollInterval, Cadence: a.cadence, MaxInflight: s.MaxInflight,
+		SNMPPort: s.SNMPPort, Logger: a.log,
 		Profiles:     a.profiles,
 		LoadProfiles: func() (*profile.Set, error) { return profile.Load(dir) },
 		Observer:     a.alerting.reach, ICMPOff: s.ICMP == config.ICMPOff,

internal/auth/auth.go

@@ -33,6 +33,10 @@ type Options struct {
 	// DevOpen turns authentication off: every request outside ingest is an admin Principal named "dev".
 	// It exists for `serve --insecure-http` with no OIDC or break-glass configured (development only).
 	DevOpen bool
+	// Install is first-install mode: no sign-in is configured yet, so only the routes the Policy marks with
+	// AllowInstall (and the browser pages) are served, to anyone, as an admin; the rest answer 503. The owner
+	// accepted this for DMZ / internal deployments; it ends for good when setup completes.
+	Install bool
 	// KeyRotateEvery is how often the cookie-signing key rotates; default 30 days.
 	KeyRotateEvery time.Duration
 }
@@ -124,6 +128,9 @@ func (a *Auth) SetTrustedProxies(cidrs []string) error {
 	return nil
 }
 
+// InstallMode reports whether the service is in first-install mode.
+func (a *Auth) InstallMode() bool { return a.opts.Install }
+
 // DevOpen reports whether authentication is off.
 func (a *Auth) DevOpen() bool { return a.opts.DevOpen }
 

internal/auth/breakglass.go

@@ -224,3 +224,30 @@ func Unlock(ctx context.Context, st Store, actor, target string) (int, error) {
 	}
 	return n, nil
 }
+
+// MinBreakGlassPassword is the shortest password the setup wizard accepts for the local admin.
+const MinBreakGlassPassword = 12
+
+// WriteBreakGlassFile writes "<user>:<bcrypt hash>" to path with mode 0600 (atomically), the format
+// BreakGlass reads. user must be non-empty and free of ":" and whitespace.
+func WriteBreakGlassFile(path, user, password string) error {
+	if user == "" || strings.ContainsAny(user, ": \t\r\n") {
+		return errors.New("auth: the break-glass user must be non-empty and contain no colon or whitespace")
+	}
+	if len(password) < MinBreakGlassPassword {
+		return fmt.Errorf("auth: the break-glass password must be at least %d characters", MinBreakGlassPassword)
+	}
+	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+	if err != nil { // also rejects passwords over 72 bytes
+		return fmt.Errorf("auth: hash password: %w", err)
+	}
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, []byte(user+":"+string(hash)+"\n"), 0o600); err != nil {
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		_ = os.Remove(tmp) // best-effort: the rename error is the one to report
+		return err
+	}
+	return nil
+}

internal/auth/cookie_secure_test.go

@@ -0,0 +1,62 @@
+package auth_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+// sessionCookieFor signs in through break-glass with the given proxy headers and returns the session cookie.
+func sessionCookieFor(t *testing.T, e *env, hdr map[string]string) *http.Cookie {
+	t.Helper()
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, e.srv.URL+"/auth/breakglass",
+		strings.NewReader(`{"username":"breakglass","password":"`+bgPassword+`"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	for k, v := range hdr {
+		req.Header.Set(k, v)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer closeBody(t, resp)
+	if resp.StatusCode != http.StatusNoContent {
+		t.Fatalf("break-glass: %d", resp.StatusCode)
+	}
+	for _, c := range resp.Cookies() {
+		if c.Name != "" && c.MaxAge > 0 {
+			return c
+		}
+	}
+	t.Fatal("no session cookie")
+	return nil
+}
+
+func TestSessionCookieIsSecureOnlyWhenTheBrowserUsesHTTPS(t *testing.T) {
+	// The test server is plain HTTP on the loopback address, which is the "trusted proxy" in the first two cases.
+	for name, tc := range map[string]struct {
+		proxies    []string
+		hdr        map[string]string
+		wantSecure bool
+	}{
+		"direct http":                         {nil, nil, false},
+		"direct http, spoofed proto":          {nil, map[string]string{"X-Forwarded-Proto": "https"}, false},
+		"trusted proxy says https":            {[]string{"127.0.0.1", "::1"}, map[string]string{"X-Forwarded-Proto": "https"}, true},
+		"trusted proxy says http":             {[]string{"127.0.0.1", "::1"}, map[string]string{"X-Forwarded-Proto": "http"}, false},
+		"trusted proxy, no proto header":      {[]string{"127.0.0.1", "::1"}, nil, false},
+		"untrusted proxy spoofing https":      {[]string{"10.0.0.0/8"}, map[string]string{"X-Forwarded-Proto": "https"}, false},
+		"untrusted proxy spoofing everything": {[]string{"10.0.0.0/8"}, map[string]string{"X-Forwarded-Proto": "https", "X-Forwarded-For": "203.0.113.9"}, false},
+	} {
+		t.Run(name, func(t *testing.T) {
+			e := newEnv(t, envOpts{proxies: tc.proxies, secure: true})
+			c := sessionCookieFor(t, e, tc.hdr)
+			if c.Secure != tc.wantSecure || !c.HttpOnly {
+				t.Fatalf("cookie Secure=%v HttpOnly=%v, want Secure=%v", c.Secure, c.HttpOnly, tc.wantSecure)
+			}
+		})
+	}
+}

internal/auth/handlers.go

@@ -52,7 +52,7 @@ func (a *Auth) handleCallback(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	a.auditLogin(r, "oidc")
-	http.SetCookie(w, cookie)
+	setCookie(w, r, cookie)
 	http.Redirect(w, r, next, http.StatusFound)
 }
 
@@ -90,7 +90,7 @@ func (a *Auth) handleBreakGlass(w http.ResponseWriter, r *http.Request) {
 	client, _ := ClientFromContext(r.Context())
 	cookie, err := a.BreakGlass.Login(r.Context(), client, body.Username, body.Password)
 	if err == nil {
-		http.SetCookie(w, cookie)
+		setCookie(w, r, cookie)
 		w.WriteHeader(http.StatusNoContent)
 		return
 	}
@@ -111,7 +111,7 @@ func (a *Auth) handleBreakGlassBasic(w http.ResponseWriter, r *http.Request) {
 		a.breakGlassFailed(w, err, true)
 		return
 	}
-	http.SetCookie(w, cookie)
+	setCookie(w, r, cookie)
 	http.Redirect(w, r, safeNext(returnParam(r)), http.StatusFound)
 }
 
@@ -151,7 +151,7 @@ func (a *Auth) handleLogout(w http.ResponseWriter, r *http.Request) {
 		writeProblem(w, http.StatusInternalServerError, "logout failed")
 		return
 	}
-	http.SetCookie(w, a.Sessions.ClearCookie())
+	setCookie(w, r, a.Sessions.ClearCookie())
 	if a.OIDC != nil && p.Via == ViaSession {
 		if err := a.OIDC.EndSession(r.Context(), idp); err != nil {
 			a.opts.Logger.Warn("IDP end_session failed", slog.Any("err", err))

internal/auth/helpers_test.go

@@ -62,6 +62,7 @@ type env struct {
 
 type envOpts struct {
 	proxies   []string
+	secure    bool // keep the Secure cookie attribute (the default test server is plain http, so it is dropped)
 	noOIDC    bool
 	noEndSess bool
 	adminGrp  string
@@ -94,7 +95,7 @@ func newEnv(t *testing.T, o envOpts) *env {
 	if o.session.Idle == 0 {
 		o.session.Idle = time.Hour
 	}
-	o.session.InsecureCookies = true // the test server is plain http; Secure is asserted separately
+	o.session.InsecureCookies = !o.secure // the test server is plain http; Secure is asserted separately
 	opts := auth.Options{
 		Store: e.st, Clock: clk, Alarms: e.alarms, Session: o.session, TrustedProxies: o.proxies,
 		BreakGlass:     auth.BreakGlassConfig{File: e.bgFile},

internal/auth/install_test.go

@@ -0,0 +1,44 @@
+package auth_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/v10networks/nmon/internal/auth"
+	"github.com/v10networks/nmon/internal/clock"
+	"github.com/v10networks/nmon/internal/testutil"
+)
+
+func TestInstallModeServesOnlyAllowedRoutesAndPagesAsAdmin(t *testing.T) {
+	clk := clock.Real()
+	pol := auth.NewPolicy().Public("GET /healthz").AllowInstall("GET /api/v1/ok", "POST /api/v1/ok")
+	svc, err := auth.New(context.Background(), auth.Options{Store: testutil.OpenStore(t, clk), Policy: pol, Install: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var who string
+	h := svc.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if p, ok := auth.FromContext(r.Context()); ok {
+			who = p.Actor() + "/" + string(p.Role)
+		}
+	}))
+	for _, tc := range []struct {
+		method, path string
+		want         int
+	}{
+		{"GET", "/api/v1/ok", 200}, {"POST", "/api/v1/ok", 200}, {"PUT", "/api/v1/ok", 503}, {"GET", "/api/v1/other", 503},
+		{"GET", "/metrics", 503}, {"GET", "/devices", 200}, {"GET", "/healthz", 200}, {"POST", "/devices", 503},
+	} {
+		rec := httptest.NewRecorder()
+		who = ""
+		h.ServeHTTP(rec, httptest.NewRequest(tc.method, tc.path, nil))
+		if rec.Code != tc.want {
+			t.Errorf("%s %s = %d, want %d", tc.method, tc.path, rec.Code, tc.want)
+		}
+		if tc.want == 200 && tc.path != "/healthz" && who != "setup/admin" {
+			t.Errorf("%s %s ran as %q, want setup/admin", tc.method, tc.path, who)
+		}
+	}
+}