6 Commits
m2 ... m5

Author SHA1 Message Date
4a57eac1ed M4 and M5 gates passed: gate logs
Some checks failed
ci / lint (push) Has been cancelled
ci / test (ubuntu-24.04-arm) (push) Has been cancelled
ci / test (ubuntu-latest) (push) Has been cancelled
ci / postgres (push) Has been cancelled
ci / openapi (push) Has been cancelled
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
2026-10-23 00:39:45 +00:00
b3aae800bf M4 integration (alerting wired, alerts API/UI, maintenance, dependency hold, redundant-topology gate) + M5 (rollup, ingest, Proxmox, checks, p95, groups, longq, gate) + M6 library packages (auth, backup, diskguard, importcsv, keyrotate)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
2026-10-23 00:10:25 +00:00
ba960474bc test-pg.sh: unique container and Docker-assigned port so concurrent runs never collide
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
2026-10-22 21:55:03 +00:00
d3ec85a19c Approved deviation: skip the 2,000-device loadtest; M6 gate is the viewer-403 check
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
2026-10-22 21:48:34 +00:00
b740611136 Docs: web push deferred to v2
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
2026-10-22 21:22:55 +00:00
f85c3ef781 M3: Go API per UI contract, embedded UI, openapi compat check, nmon-devtools, live gate + screenshots; M4 WP1 reachability and dependency suppression
Some checks failed
ci / lint (push) Has been cancelled
ci / test (ubuntu-24.04-arm) (push) Has been cancelled
ci / test (ubuntu-latest) (push) Has been cancelled
ci / postgres (push) Has been cancelled
ci / openapi (push) Has been cancelled
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
2026-10-22 20:34:14 +00:00
526 changed files with 50431 additions and 1298 deletions

3
.gitignore vendored
View File

@@ -1,5 +1,6 @@
/nmon
/nmon-arm64
/nmon-devtools
/internal/vmsup/bin/*.tar.gz
/internal/vmsup/bin/victoria-metrics-*
web/node_modules/
@@ -8,3 +9,5 @@ web/dist/
*.test
coverage.out
/nmon-librenms-import
/internal/webui/dist/*
!/internal/webui/dist/.gitkeep

View File

@@ -107,6 +107,26 @@ linters:
desc: collection paths share code via a lower package, not each other
- pkg: github.com/v10networks/nmon/internal/traps
desc: collection paths share code via a lower package, not each other
devtools-not-in-nmon:
# The product binary never links dev/test machinery. `nmon profile record` needs only the
# .snmprec format, which lives in internal/snmprec.
files: ["**/cmd/nmon/**", "!$test"]
deny:
- pkg: github.com/v10networks/nmon/internal/simagent
desc: the simulator belongs to cmd/nmon-devtools and tests, never the shipped binary
- pkg: github.com/v10networks/nmon/internal/fixturegen
desc: the fixture generator belongs to cmd/nmon-devtools and tests, never the shipped binary
- pkg: github.com/v10networks/nmon/internal/fixtures
desc: fixtures are test data, never the shipped binary
- pkg: github.com/v10networks/nmon/internal/testutil
desc: test helpers are never in the shipped binary
devtools-no-core:
files: ["**/cmd/nmon-devtools/**"]
deny:
- pkg: github.com/v10networks/nmon/internal/app
desc: nmon-devtools talks to a running nmon over HTTP
- pkg: github.com/v10networks/nmon/internal/api
desc: nmon-devtools talks to a running nmon over HTTP
importer-not-in-nmon:
files: ["**/cmd/nmon/**"]
deny:

View File

@@ -1,24 +1,33 @@
GO ?= go
TAGS ?= vmembed
LDFLAGS := -s -w
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
BUILD ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown)-$(shell date -u +%Y%m%d)
LDFLAGS := -s -w -X github.com/v10networks/nmon/internal/buildinfo.Version=$(VERSION) -X github.com/v10networks/nmon/internal/buildinfo.Build=$(BUILD)
BIN := nmon
PG_DSN ?= postgres://nmon:nmon@localhost:55432/nmon?sslmode=disable
.PHONY: all lint lint-go lint-web build importer build-arm64 build-noembed fetch-vm web test test-integration test-pg test-v6only vet gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6 clean
.PHONY: devtools all lint lint-go lint-web build importer build-arm64 build-noembed fetch-vm web test test-integration test-pg test-v6only vet gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6 clean
all: build
fetch-vm:
./scripts/fetch-vm.sh
# The UI is built first when web/ exists so embed.FS picks it up.
# The UI is built first, then copied into internal/webui/dist for go:embed (git-ignored).
web:
@if [ -f web/package.json ]; then cd web && npm ci --no-audit --no-fund && npm run build; fi
@if [ -f web/package.json ]; then \
cd web && { [ -d node_modules ] || npm ci --no-audit --no-fund; } && npm run build && \
cd .. && rm -rf internal/webui/dist/* && cp -r web/dist/. internal/webui/dist/ && touch internal/webui/dist/.gitkeep; \
fi
build: fetch-vm
build: fetch-vm web
CGO_ENABLED=0 $(GO) build -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(BIN) ./cmd/nmon
build-arm64: fetch-vm
# Developer tools (backfill, seed-demo) are a separate binary that is never shipped.
devtools:
CGO_ENABLED=0 $(GO) build -o nmon-devtools ./cmd/nmon-devtools
build-arm64: fetch-vm web
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(GO) build -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(BIN)-arm64 ./cmd/nmon
importer:
@@ -59,4 +68,4 @@ gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6: gate-m%:
./scripts/gate-m$*.sh
clean:
rm -f $(BIN) $(BIN)-arm64 nmon-librenms-import
rm -f $(BIN) $(BIN)-arm64 nmon-devtools nmon-librenms-import

View File

@@ -0,0 +1,162 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"math"
"net/http"
"net/url"
"strings"
"time"
"github.com/v10networks/nmon/internal/metrics"
)
// backfillConfig is what one `devtools backfill` run writes, for every interface in ifaces.
type backfillConfig struct {
hotURL string
device string
ifaces []string
days int
step time.Duration
pattern string
meanBps float64 // bits/s; 0 derives the mean from the live rate, else a stable per-interface value
seed uint64
hc *http.Client
logf func(format string, a ...any)
}
func backfillCmd(ctx context.Context, args []string) error {
cfg := backfillConfig{hc: &http.Client{Timeout: 2 * time.Minute}, logf: func(f string, a ...any) { fmt.Printf(f+"\n", a...) }}
fs := devFlagSet("backfill")
var ifaces stringList
var mbps float64
fs.StringVar(&cfg.hotURL, "hot-url", "", "base URL of the hot VictoriaMetrics (required; see \"metric stores up\" in the serve log)")
fs.StringVar(&cfg.device, "device", "", "device slug (the series' device label)")
fs.Var(&ifaces, "iface", "interface ULID; repeat the flag or separate with commas")
fs.IntVar(&cfg.days, "days", 31, "days of history to write, ending where the live data begins")
fs.DurationVar(&cfg.step, "step", 60*time.Second, "sample spacing")
fs.StringVar(&cfg.pattern, "pattern", patternDiurnal, "traffic shape: diurnal or flat")
fs.Float64Var(&mbps, "mean-mbps", 0, "mean rate in Mbit/s; 0 follows the live rate when there is one")
fs.Uint64Var(&cfg.seed, "seed", 1, "noise seed")
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
cfg.ifaces, cfg.meanBps = ifaces, mbps*1e6
if err := cfg.validate(); err != nil {
return err
}
for _, id := range cfg.ifaces {
if err := cfg.backfillIface(ctx, id); err != nil {
return fmt.Errorf("iface %s: %w", id, err)
}
}
return nil
}
func (c *backfillConfig) validate() error {
switch {
case c.hotURL == "":
return errors.New("--hot-url is required")
case c.device == "" || len(c.ifaces) == 0:
return errors.New("--device and at least one --iface are required")
case c.days < 1 || c.step < time.Second:
return errors.New("--days must be at least 1 and --step at least 1s")
case c.pattern != patternDiurnal && c.pattern != patternFlat:
return fmt.Errorf("unknown --pattern %q (diurnal or flat)", c.pattern)
}
c.hotURL = strings.TrimRight(c.hotURL, "/")
return nil
}
// liveSeries is what the hot store already holds for one series: the first
// sample (where history must end and which value it must join) and the live rate.
type liveSeries struct {
ts time.Time
val float64
rate float64 // units per second between the first two samples; 0 when unknown
ok bool
}
// backfillIface writes in/out octet counters and the oper status for one interface.
func (c *backfillConfig) backfillIface(ctx context.Context, iface string) error {
n := c.days * int(24*time.Hour/c.step)
var samples []metrics.Sample
var end time.Time
for d, dir := range []struct {
metric string
salt uint64
}{{"ifHCInOctets", 0x1111}, {"ifHCOutOctets", 0x2222}} {
live, err := c.probe(ctx, dir.metric, iface)
if err != nil {
return err
}
if d == 0 {
end = c.endTime(live)
}
samples = append(samples, c.counter(dir.metric, iface, end, n, live, c.seed^dir.salt)...)
}
status := 1.0
if live, err := c.probe(ctx, "ifOperStatus", iface); err == nil && live.ok {
status = live.val
}
for i := 0; i < n; i++ {
ts := end.Add(-time.Duration(n-1-i) * c.step)
samples = append(samples, metrics.New("ifOperStatus", status, ts, metrics.LabelDevice, c.device, metrics.LabelIface, iface))
}
if err := c.send(ctx, samples); err != nil {
return err
}
c.logf("backfilled %s/%s: %d days at %s (%d samples)", c.device, iface, c.days, c.step, len(samples))
return nil
}
// endTime is one step before the first live sample, or one step ago when there is none.
func (c *backfillConfig) endTime(live liveSeries) time.Time {
if live.ok {
return live.ts.Add(-c.step).Truncate(time.Millisecond)
}
return time.Now().Add(-c.step).Truncate(time.Millisecond)
}
// counter builds one direction's history ending at end. When the live counter
// is known and large enough, the history is shifted so its last value is where
// the live counter was one step earlier and the join shows no reset.
func (c *backfillConfig) counter(metric, iface string, end time.Time, n int, live liveSeries, salt uint64) []metrics.Sample {
mean := c.meanBps / 8
if mean == 0 && live.rate > 0 {
mean = live.rate
}
if mean == 0 {
mean = defaultMeanBytesPerSec(iface, salt)
}
start := end.Add(-time.Duration(n-1) * c.step)
vals := counterSeries(c.pattern, start, n, c.step, mean, salt)
var offset float64
if live.ok {
offset = live.val - mean*c.step.Seconds() - vals[n-1]
if offset < 0 {
c.logf("warning: %s %s: live counter %.3g is below the %.3g bytes of history; the join will look like a counter reset",
c.device, metric, live.val, vals[n-1])
offset = 0
}
}
out := make([]metrics.Sample, n)
for i, v := range vals {
ts := start.Add(time.Duration(i) * c.step)
out[i] = metrics.New(metric, math.Floor(v+offset), ts, metrics.LabelDevice, c.device, metrics.LabelIface, iface)
}
return out
}
// probe reads the earliest samples of metric{iface=...} from the hot store.
func (c *backfillConfig) probe(ctx context.Context, metric, iface string) (liveSeries, error) {
q := url.Values{"match[]": {fmt.Sprintf(`%s{iface=%q}`, metric, iface)}}
var raw bytes.Buffer
if err := c.get(ctx, c.hotURL+"/api/v1/export?"+q.Encode(), &raw); err != nil {
return liveSeries{}, err
}
return parseExport(raw.Bytes())
}

View File

@@ -0,0 +1,80 @@
package main
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"time"
"github.com/v10networks/nmon/internal/longq"
"github.com/v10networks/nmon/internal/metrics"
"github.com/v10networks/nmon/internal/rollup"
)
// backfillLongCmd writes rollup series straight into the long store, as if the rollup job had been running
// for years: hourly average and maximum rates for --months months and 5-minute rates for --days days, flat
// at the given rates so a gate can assert exact values. The data ends --end-offset ago, leaving the recent
// hours to the real rollup job (its repair writes higher versions, which win).
func backfillLongCmd(ctx context.Context, args []string) error {
fs := devFlagSet("backfill-long")
var ifaces stringList
long := fs.String("long-url", "", "base URL of the long VictoriaMetrics (see \"metric stores up\" in the serve log)")
device := fs.String("device", "", "device slug (the series' device label)")
fs.Var(&ifaces, "iface", "interface ULID; repeat the flag or separate with commas")
months := fs.Int("months", 25, "months of hourly rollups to write")
days := fs.Int("days", 70, "days of 5-minute rates to write")
offset := fs.Duration("end-offset", 3*time.Hour, "stop this long before now")
in := fs.Float64("in-bps", 8e6, "constant inbound rate")
out := fs.Float64("out-bps", 2e6, "constant outbound rate")
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
if *long == "" || *device == "" || len(ifaces) == 0 {
return fmt.Errorf("backfill-long needs --long-url, --device and --iface: %w", errUsage)
}
c := &backfillConfig{hotURL: strings.TrimRight(*long, "/"), hc: &http.Client{Timeout: 5 * time.Minute}}
end := time.Now().Add(-*offset).Truncate(time.Hour)
for _, id := range ifaces {
s := longSamples(*device, id, end, *months, *days, map[string]float64{"in": *in, "out": *out})
if err := c.send(ctx, s); err != nil {
return errors.Join(fmt.Errorf("iface %s", id), err)
}
fmt.Printf("backfilled long %s/%s: %d months hourly, %d days 5m (%d samples)\n", *device, id, *months, *days, len(s))
}
return c.flush(ctx)
}
// flush makes what was just imported searchable at once: VictoriaMetrics shows new rows only after its
// buffers flush, and a gate that queries right after the import would otherwise see a partial history.
func (c *backfillConfig) flush(ctx context.Context) error {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.hotURL+"/internal/force_flush", http.NoBody)
if err != nil {
return err
}
resp, err := c.hc.Do(req)
if err != nil {
return err
}
_ = resp.Body.Close() // nothing in the reply matters
return nil
}
// longSamples builds the hourly series from end back months months and the 5-minute series from end back days.
func longSamples(device, iface string, end time.Time, months, days int, rate map[string]float64) []metrics.Sample {
var out []metrics.Sample
labels := func(dir string, ws time.Time) []string {
return []string{"device", device, "iface", iface, "dir", dir, "v", rollup.VersionLabel(ws, 1)}
}
for dir, bps := range rate {
for h := end; h.After(end.AddDate(0, -months, 0)); h = h.Add(-time.Hour) {
out = append(out, metrics.New(longq.MetricRateAvg1h, bps, h, labels(dir, h)...),
metrics.New(longq.MetricRateMax1h, bps*1.25, h, labels(dir, h)...))
}
for t := end; t.After(end.AddDate(0, 0, -days)); t = t.Add(-5 * time.Minute) {
out = append(out, metrics.New(longq.MetricRate5m, bps, t, labels(dir, t)...))
}
}
return out
}

130
cmd/nmon-devtools/fleet.go Normal file
View File

@@ -0,0 +1,130 @@
package main
import (
"fmt"
"strconv"
"strings"
"github.com/gosnmp/gosnmp"
"github.com/v10networks/nmon/internal/fixturegen"
"github.com/v10networks/nmon/internal/profile"
"github.com/v10networks/nmon/internal/snmprec"
)
// demoDevice is one simulated device of `devtools seed-demo`.
type demoDevice struct {
name, site, role string
kind string // counters, big, router, ups, v1, down: what the gate and screenshots look for
rec *snmprec.Recording
community string
version string // v2c unless set
hc bool // 64-bit counters, so history can be back-filled
}
// counterBase is added to every simulated 64-bit octet counter so that the
// back-filled history (about 3e13 bytes for 31 days at 100 Mbit/s) fits below
// the live counter and the join shows no reset.
const counterBase = 100_000_000_000_000
const (
oidHCIn = "1.3.6.1.2.1.31.1.1.1.6."
oidHCOut = "1.3.6.1.2.1.31.1.1.1.10."
oidUp = "1.3.6.1.2.1.1.3.0"
)
// animate returns a copy of rec whose 64-bit octet counters advance at a steady
// per-port rate from counterBase and whose uptime ticks, so successive polls see
// traffic. Counters that already advance keep their rate and only gain the base.
func animate(rec *snmprec.Recording) *snmprec.Recording {
recs := append([]snmprec.Record(nil), rec.Records()...)
for i := range recs {
r := &recs[i]
switch {
case r.Type == gosnmp.Counter64 && (strings.HasPrefix(r.OID, oidHCIn) || strings.HasPrefix(r.OID, oidHCOut)):
recs[i] = animateCounter(*r)
case r.OID == oidUp && r.Dyn == nil:
if v, ok := r.Value.(uint32); ok {
r.Dyn, r.Value = &snmprec.Dynamic{Rate: 100, Initial: uint64(v)}, nil
}
}
}
return snmprec.NewRecording(recs)
}
func animateCounter(r snmprec.Record) snmprec.Record {
if r.Dyn != nil {
d := *r.Dyn
d.Initial += counterBase
r.Dyn = &d
return r
}
k, _ := strconv.Atoi(r.OID[strings.LastIndexByte(r.OID, '.')+1:])
dir := 1
if strings.HasPrefix(r.OID, oidHCOut) {
dir = 2
}
mbps := 1 + (k*(31+7*dir))%90
r.Dyn = &snmprec.Dynamic{Rate: float64(mbps) * 125_000, Initial: counterBase + uint64(k)*1_000_000_000}
r.Value = nil
return r
}
// profileDevice builds a device that a shipped profile matches, with traffic.
func profileDevice(set *profile.Set, prof, name, site, role, kind string, ports int) (demoDevice, error) {
spec, ok := fixturegen.SpecFor(prof)
if !ok {
return demoDevice{}, fmt.Errorf("no fixture spec for profile %q", prof)
}
res, ok := set.Lookup(prof)
if !ok {
return demoDevice{}, fmt.Errorf("no profile %q", prof)
}
spec.SysName = name
if ports > 0 {
spec.Ports = ports
}
rec, err := fixturegen.FromProfile(res, spec)
if err != nil {
return demoDevice{}, fmt.Errorf("%s: %w", name, err)
}
return demoDevice{name: name, site: site, role: role, kind: kind, rec: animate(rec), community: "public", hc: true}, nil
}
// buildFleet is the small network used for live screenshots: a 48-port switch,
// routers, a firewall, a UPS, one 32-bit device, the counters fixture and one
// device whose agent is stopped after seeding, so it reads as down.
func buildFleet(set *profile.Set, countersRec string) ([]demoDevice, error) {
counters, err := snmprec.Load(countersRec)
if err != nil {
return nil, err
}
fleet := []demoDevice{{
name: "yyz1-edge1", site: "yyz1", role: "edge", kind: "counters", rec: animate(counters), community: "public", hc: true,
}}
for _, p := range []struct {
prof, name, site, role, kind string
ports int
}{
{"cisco-ios", "yyz1-acc01", "yyz1", "access", "big", 48},
{"cisco-ios", "yyz1-acc02", "yyz1", "access", "router", 24},
{"cisco-ios", "ott1-acc02", "ott1", "access", "down", 24}, // its agent is stopped once seeding is done
{"juniper-junos", "yyz1-core1", "yyz1", "core", "router", 0},
{"arista-eos", "yul1-core1", "yul1", "core", "router", 0},
{"mikrotik-routeros", "ott1-gw1", "ott1", "gateway", "router", 0},
{"fortigate", "yyz1-fw1", "yyz1", "firewall", "router", 0},
{"apc-ups", "yyz1-ups1", "yyz1", "power", "ups", 0},
} {
d, err := profileDevice(set, p.prof, p.name, p.site, p.role, p.kind, p.ports)
if err != nil {
return nil, err
}
fleet = append(fleet, d)
}
// 32-bit counters on 1 Gbit/s ports: the API flags them unreliable and the UI never graphs them.
v1 := fixturegen.Generic(fixturegen.Options{Ports: 12, SysName: "yul1-acc03", Counters: true, NamePattern: "Fa0/%d"})
fleet = append(fleet, demoDevice{
name: "yul1-acc03", site: "yul1", role: "access", kind: "v1", rec: v1, community: "nmon-v1", version: "v1",
})
return fleet, nil
}

View File

@@ -0,0 +1,163 @@
package main
import (
"bufio"
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"strings"
"time"
)
// gateDumpCmd prints what the M4 gate asserts on, one line per item, so the shell
// script can count and match lines instead of parsing JSON.
//
// gate-dump sink --file F what the webhook sink received, in arrival order
// gate-dump alerts --api URL every alert, newest first
// gate-dump devices --api URL every device's reachability
func gateDumpCmd(ctx context.Context, args []string) error {
if len(args) == 0 {
return fmt.Errorf("gate-dump sink|alerts|devices")
}
fs := devFlagSet("gate-dump " + args[0])
file := fs.String("file", "", "sink file (sink)")
api := fs.String("api", "", "nmon API base URL (alerts, devices)")
mapFile := fs.String("map", "", "slug map written by topology-add; slugs are printed as topology names")
if err := fs.Parse(args[1:]); err != nil {
return flagErr(err)
}
names, err := readSlugMap(*mapFile)
if err != nil {
return err
}
switch args[0] {
case "sink":
return dumpSink(*file, names)
case "alerts":
return dumpAlerts(ctx, *api, names)
case "devices":
return dumpDevices(ctx, *api, names)
}
return fmt.Errorf("gate-dump: unknown %q", args[0])
}
func orDash(s []string) string {
if len(s) == 0 {
return "-"
}
return strings.Join(s, ",")
}
// dumpSink prints "rule state device parents=a,b children=N incident=ID" per request received.
func dumpSink(file string, names slugNames) error {
f, err := os.Open(file)
if err != nil {
return err
}
defer func() { _ = f.Close() }() // read only
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 1<<20), 1<<20)
for sc.Scan() {
var l sinkLine
var n struct {
Rule, State string
IncidentID string `json:"incident_id"`
Device struct{ Slug string }
Parents []string
SuppressedChildren int `json:"suppressed_children"`
}
if json.Unmarshal(sc.Bytes(), &l) != nil || json.Unmarshal(l.Body, &n) != nil {
fmt.Println("unparsable - - parents=- children=0 incident=-")
continue
}
fmt.Printf("%s %s %s parents=%s children=%d incident=%s\n", n.Rule, n.State, names.one(n.Device.Slug), orDash(names.of(n.Parents)),
n.SuppressedChildren, n.IncidentID)
}
return sc.Err()
}
func getJSON(ctx context.Context, api, path string, out any) error {
hc := &http.Client{Timeout: 30 * time.Second}
return httpJSON(ctx, hc, http.MethodGet, api+path, nil, out)
}
// dumpAlerts prints "id rule state device suppressed_by=a,b incident=ID acked=yes|no" per alert.
func dumpAlerts(ctx context.Context, api string, names slugNames) error {
var page struct {
Items []struct {
ID, Rule, State string
Device struct{ Slug string }
SuppressedBy []string `json:"suppressed_by"`
IncidentID string `json:"incident_id"`
Ack *struct{} `json:"ack"`
} `json:"items"`
}
if err := getJSON(ctx, api, "/api/v1/alerts?per_page=1000", &page); err != nil {
return err
}
for _, a := range page.Items {
acked := "no"
if a.Ack != nil {
acked = "yes"
}
fmt.Printf("%s %s %s %s suppressed_by=%s incident=%s acked=%s\n", a.ID, a.Rule, a.State, names.one(a.Device.Slug),
orDash(names.of(a.SuppressedBy)), orDash([]string{a.IncidentID}), acked)
}
return nil
}
// dumpDevices prints "slug reachability poll_state warnings=a,b" per device.
func dumpDevices(ctx context.Context, api string, names slugNames) error {
var page struct {
Items []struct {
Slug, Reachability string
PollState string `json:"poll_state"`
Warnings []string `json:"warnings"`
} `json:"items"`
}
if err := getJSON(ctx, api, "/api/v1/devices?per_page=1000", &page); err != nil {
return err
}
for _, d := range page.Items {
fmt.Printf("%s %s %s warnings=%s\n", url.PathEscape(names.one(d.Slug)), d.Reachability, d.PollState, orDash(d.Warnings))
}
return nil
}
// slugNames maps nmon slugs back to the names in the topology file.
type slugNames map[string]string
func readSlugMap(path string) (slugNames, error) {
m := slugNames{}
if path == "" {
return m, nil
}
b, err := os.ReadFile(path)
if err != nil {
return nil, err
}
for _, l := range strings.Split(string(b), "\n") {
if f := strings.Fields(l); len(f) == 2 {
m[f[0]] = f[1]
}
}
return m, nil
}
func (m slugNames) one(slug string) string {
if n, ok := m[slug]; ok {
return n
}
return slug
}
func (m slugNames) of(slugs []string) []string {
out := make([]string, len(slugs))
for i, s := range slugs {
out[i] = m.one(s)
}
return out
}

View File

@@ -0,0 +1,59 @@
package main
import (
"strings"
"testing"
"time"
)
func TestRebaseInfluxMakesTheNewestLineNowAndKeepsSpacing(t *testing.T) {
in := "m,host=a v=1 1000000000000000000\n# c\nm,host=a v=2 1000000060000000000\n"
now := time.Unix(2_000_000_000, 0)
out, n, err := rebaseInflux(in, now)
if err != nil || n != 2 {
t.Fatalf("n=%d err=%v", n, err)
}
want := "m,host=a v=1 1999999940000000000\nm,host=a v=2 2000000000000000000\n"
if out != want {
t.Fatalf("got %q want %q", out, want)
}
if _, _, err := rebaseInflux("\n# only comments\n", now); err == nil {
t.Fatal("empty input accepted")
}
if !strings.Contains(out, "host=a") {
t.Fatal("tags lost")
}
}
func TestProxmoxMockAuthShape(t *testing.T) {
for h, want := range map[string]bool{
"PVEAPIToken=root@pam!nmon=s3cret": true, "PVEAPIToken=root@pam!nmon=": false, "Bearer x": false, "": false,
} {
if validPVEAuth(h, "") != want {
t.Errorf("%q: want %v", h, want)
}
}
if validPVEAuth("PVEAPIToken=a@b!c=other", "s3cret") {
t.Error("wrong secret accepted")
}
}
func TestLongSamplesCarryVersionedHourlyAndFiveMinuteSeries(t *testing.T) {
end := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC)
s := longSamples("d", "I", end, 1, 1, map[string]float64{"in": 8e6})
var h, f int
for _, x := range s {
switch x.Name {
case "nmon_if_rate_bps_5m":
f++
default:
h++
}
}
if f != 288 || h < 2*28*24 {
t.Fatalf("5m=%d hourly=%d", f, h)
}
if s[0].Labels[3].Name != "v" || s[0].Labels[3].Value != "202610001" {
t.Fatalf("labels %+v", s[0].Labels)
}
}

135
cmd/nmon-devtools/main.go Normal file
View File

@@ -0,0 +1,135 @@
// Command nmon-devtools holds developer helpers for the gates and for screenshots: backfill (synthetic
// history into the hot store) and seed-demo (a simulated network added to a running nmon). It talks to nmon
// only through its HTTP API and to the hot store through --hot-url, and is never shipped (docs/dev.md).
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
)
var errUsage = errors.New("usage: nmon-devtools backfill|seed-demo|topology-agents|topology-add|webhook-sink|gate-dump|" +
"proxmox-mock|push-replay|backfill-long [flags]" +
" (see docs/dev.md)")
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Args[1:]); err != nil {
fmt.Fprintln(os.Stderr, "nmon-devtools:", err)
os.Exit(1)
}
}
func run(ctx context.Context, args []string) error {
if len(args) == 0 {
return errUsage
}
switch args[0] {
case "backfill":
return backfillCmd(ctx, args[1:])
case "seed-demo":
return seedDemoCmd(ctx, args[1:])
case "topology-agents":
return topologyAgentsCmd(ctx, args[1:])
case "topology-add":
return topologyAddCmd(ctx, args[1:])
case "webhook-sink":
return webhookSinkCmd(ctx, args[1:])
case "gate-dump":
return gateDumpCmd(ctx, args[1:])
case "proxmox-mock":
return proxmoxMockCmd(ctx, args[1:])
case "push-replay":
return pushReplayCmd(ctx, args[1:])
case "backfill-long":
return backfillLongCmd(ctx, args[1:])
default:
return fmt.Errorf("unknown command %q: %w", args[0], errUsage)
}
}
// flagErr turns flag.ErrHelp into a clean exit.
func flagErr(err error) error {
if errors.Is(err, flag.ErrHelp) {
os.Exit(0)
}
return err
}
// stringList is a repeatable string flag.
type stringList []string
func (s *stringList) String() string { return strings.Join(*s, ",") }
func (s *stringList) Set(v string) error {
for _, p := range strings.Split(v, ",") {
if p = strings.TrimSpace(p); p != "" {
*s = append(*s, p)
}
}
return nil
}
// devFlagSet is a FlagSet that prints its own usage to stderr.
func devFlagSet(name string) *flag.FlagSet {
fs := flag.NewFlagSet("nmon-devtools "+name, flag.ContinueOnError)
fs.SetOutput(os.Stderr)
return fs
}
// httpJSON sends an optional JSON body and decodes a 2xx JSON reply into out (nil to discard it).
func httpJSON(ctx context.Context, hc *http.Client, method, url string, in, out any) error {
var body io.Reader
if in != nil {
b, err := json.Marshal(in)
if err != nil {
return err
}
body = strings.NewReader(string(b))
}
req, err := http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
return err
}
if in != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := hc.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }() // body read to the end below
data, err := io.ReadAll(io.LimitReader(resp.Body, 64<<20))
if err != nil {
return err
}
if resp.StatusCode/100 != 2 {
return fmt.Errorf("%s %s: %s: %s", method, url, resp.Status, strings.TrimSpace(string(data[:min(len(data), 300)])))
}
if out == nil {
return nil
}
return json.Unmarshal(data, out)
}
func sleepCtx(ctx context.Context, d time.Duration) bool {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return false
case <-t.C:
return true
}
}

View File

@@ -0,0 +1,69 @@
package main
import (
"math"
"time"
)
// Synthetic traffic shapes for `nmon devtools backfill`. Everything here is a
// pure function of its inputs, so a back-fill is reproducible and testable.
const (
patternDiurnal = "diurnal"
patternFlat = "flat"
)
// splitmix64 is a tiny deterministic hash used for noise; math/rand would make
// two runs differ and a global source would be shared state.
func splitmix64(x uint64) uint64 {
x += 0x9e3779b97f4a7c15
x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9
x = (x ^ (x >> 27)) * 0x94d049bb133111eb
return x ^ (x >> 31)
}
// unit maps a hash to [0, 1).
func unit(h uint64) float64 { return float64(h>>11) / (1 << 53) }
// rateFactor is the multiple of the mean rate at t: 1 for "flat"; for
// "diurnal" an evening peak (about 20:00 UTC), an early-morning trough, quieter
// weekends, per-sample noise and the odd short burst.
func rateFactor(pattern string, t time.Time, salt uint64) float64 {
if pattern == patternFlat {
return 1
}
h := float64(t.Hour()) + float64(t.Minute())/60
f := 1 + 0.6*math.Cos(2*math.Pi*(h-20)/24)
if wd := t.Weekday(); wd == time.Saturday || wd == time.Sunday {
f *= 0.8
}
slot := uint64(t.Unix() / 60)
f *= 0.88 + 0.24*unit(splitmix64(salt^slot))
if unit(splitmix64(salt^(slot/4)^0xb5ad4eceda1ce2a9)) < 0.01 { // ~1% of 4-minute slots burst
f *= 2.5
}
return math.Max(f, 0.05)
}
// counterSeries returns n cumulative octet-counter values, one per step from
// start, beginning at 0 and never decreasing. mean is bytes per second.
func counterSeries(pattern string, start time.Time, n int, step time.Duration, mean float64, salt uint64) []float64 {
out := make([]float64, n)
var total float64
for i := 1; i < n; i++ {
t := start.Add(time.Duration(i) * step)
total += mean * rateFactor(pattern, t, salt) * step.Seconds()
out[i] = math.Floor(total)
}
return out
}
// defaultMeanBytesPerSec is a stable per-interface mean (2 to 60 Mbit/s) for when
// the live rate is unknown.
func defaultMeanBytesPerSec(iface string, salt uint64) float64 {
h := salt
for _, c := range []byte(iface) {
h = splitmix64(h ^ uint64(c))
}
return (2 + 58*unit(h)*unit(h>>7)) * 125_000
}

View File

@@ -0,0 +1,56 @@
package main
import (
"testing"
"time"
)
func TestCounterSeriesIsMonotonicAndDeterministic(t *testing.T) {
start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
a := counterSeries(patternDiurnal, start, 3*24*60, time.Minute, 1_000_000, 7)
b := counterSeries(patternDiurnal, start, 3*24*60, time.Minute, 1_000_000, 7)
for i := range a {
if a[i] != b[i] {
t.Fatalf("not deterministic at %d", i)
}
if i > 0 && a[i] < a[i-1] {
t.Fatalf("decreased at %d", i)
}
}
if a[0] != 0 {
t.Fatalf("starts at %v, want 0", a[0])
}
meanRate := a[len(a)-1] / (float64(len(a)-1) * 60)
if meanRate < 0.8e6 || meanRate > 1.4e6 {
t.Fatalf("mean rate %.0f B/s drifted from 1e6", meanRate)
}
}
func TestDiurnalEveningBusierThanNight(t *testing.T) {
// Monday, so the weekend factor does not interfere. Averaged over many days the noise washes out.
var night, evening float64
for d := 0; d < 28; d++ {
day := time.Date(2026, 9, 7, 0, 0, 0, 0, time.UTC).AddDate(0, 0, d)
if wd := day.Weekday(); wd == time.Saturday || wd == time.Sunday {
continue
}
night += rateFactor(patternDiurnal, day.Add(5*time.Hour), 1)
evening += rateFactor(patternDiurnal, day.Add(20*time.Hour), 1)
}
if evening < 2*night {
t.Fatalf("evening %.2f is not clearly busier than night %.2f", evening, night)
}
}
func TestFlatPatternIsConstant(t *testing.T) {
if f := rateFactor(patternFlat, time.Now(), 3); f != 1 {
t.Fatalf("flat factor %v", f)
}
}
func TestDefaultMeanIsStableAndBounded(t *testing.T) {
m := defaultMeanBytesPerSec("01abc", 1)
if m != defaultMeanBytesPerSec("01abc", 1) || m < 2*125_000 || m > 60*125_000 {
t.Fatalf("mean %v", m)
}
}

View File

@@ -0,0 +1,75 @@
package main
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"os"
"strings"
"time"
)
// proxmoxMockCmd serves a recorded cluster/resources response the way a Proxmox VE API does, for the M5 gate:
// GET /api2/json/cluster/resources with "Authorization: PVEAPIToken=<id>=<secret>". Any other path is a 404 and
// a request without that header shape is a 401, so a client that forgot the secret fails like the real thing.
func proxmoxMockCmd(ctx context.Context, args []string) error {
fs := devFlagSet("proxmox-mock")
listen := fs.String("listen", "[::1]:0", "listen address")
file := fs.String("file", "", "recorded cluster/resources JSON (testdata/proxmox/cluster-resources.json)")
secret := fs.String("secret", "", "token secret to require (empty: any non-empty secret)")
portFile := fs.String("port-file", "", "file that receives the bound port")
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
body, err := os.ReadFile(*file)
if err != nil || *file == "" {
return fmt.Errorf("proxmox-mock needs --file: %w", errors.Join(err, errUsage))
}
ln, err := net.Listen("tcp", *listen)
if err != nil {
return err
}
if err := writePort(*portFile, ln); err != nil {
return err
}
mux := http.NewServeMux()
mux.HandleFunc("GET /api2/json/cluster/resources", func(w http.ResponseWriter, r *http.Request) {
if !validPVEAuth(r.Header.Get("Authorization"), *secret) {
http.Error(w, `{"data":null}`, http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(body) // a client that hung up has nobody to tell
})
srv := &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second}
go func() { <-ctx.Done(); _ = srv.Close() }()
fmt.Printf("proxmox-mock listening on %s\n", ln.Addr())
if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}
// validPVEAuth accepts "PVEAPIToken=user@realm!id=secret" with the wanted secret (or any, when want is empty).
func validPVEAuth(h, want string) bool {
rest, ok := strings.CutPrefix(h, "PVEAPIToken=")
if !ok {
return false
}
_, got, ok := strings.Cut(rest, "=")
return ok && got != "" && (want == "" || got == want)
}
// writePort records the bound port for a shell script to read.
func writePort(file string, ln net.Listener) error {
if file == "" {
return nil
}
tcp, ok := ln.Addr().(*net.TCPAddr)
if !ok {
return fmt.Errorf("listener is not TCP: %s", ln.Addr())
}
return os.WriteFile(file, []byte(fmt.Sprint(tcp.Port)), 0o600)
}

View File

@@ -0,0 +1,95 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"os"
"strconv"
"strings"
"time"
)
// pushReplayCmd sends a recorded Influx line-protocol file to a running nmon's POST /write, shifting every
// timestamp so the newest line is "now": a recording is days old by the time a gate replays it, and ingest
// refuses timestamps far from the clock.
func pushReplayCmd(ctx context.Context, args []string) error {
fs := devFlagSet("push-replay")
url := fs.String("url", "", "ingest URL, for example http://127.0.0.1:8443/write")
token := fs.String("token", "", "push token secret (omit to test the unauthenticated case)")
file := fs.String("file", "", "Influx line-protocol file")
extra := fs.String("append", "", "one more line to send after the file (a raw line, rebased like the others)")
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
raw, err := os.ReadFile(*file)
if err != nil || *url == "" {
return fmt.Errorf("push-replay needs --url and --file: %w", errors.Join(err, errUsage))
}
body, n, err := rebaseInflux(string(raw)+"\n"+*extra, time.Now())
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, *url, bytes.NewReader([]byte(body)))
if err != nil {
return err
}
if *token != "" {
req.Header.Set("Authorization", "Bearer "+*token)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }() // body read below
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 500))
fmt.Printf("%d lines -> HTTP %d %s\n", n, resp.StatusCode, strings.TrimSpace(string(msg)))
if resp.StatusCode/100 != 2 {
return fmt.Errorf("push refused: HTTP %d", resp.StatusCode)
}
return nil
}
// rebaseInflux shifts the nanosecond timestamp at the end of every line by the same amount so the newest one
// equals now. Lines without a timestamp, blank lines and comments pass through.
func rebaseInflux(text string, now time.Time) (string, int, error) {
lines := strings.Split(text, "\n")
var newest int64
for _, l := range lines {
if ts, ok := lineTS(l); ok {
newest = max(newest, ts)
}
}
shift := now.UnixNano() - newest
var out strings.Builder
n := 0
for _, l := range lines {
l = strings.TrimSpace(l)
if l == "" || strings.HasPrefix(l, "#") {
continue
}
n++
if ts, ok := lineTS(l); ok {
l = l[:strings.LastIndexByte(l, ' ')+1] + strconv.FormatInt(ts+shift, 10)
}
out.WriteString(l + "\n")
}
if n == 0 {
return "", 0, errors.New("no lines to send")
}
return out.String(), n, nil
}
// lineTS returns the trailing nanosecond timestamp of an Influx line, if it has one.
func lineTS(l string) (int64, bool) {
l = strings.TrimSpace(l)
i := strings.LastIndexByte(l, ' ')
if l == "" || strings.HasPrefix(l, "#") || i < 0 {
return 0, false
}
ts, err := strconv.ParseInt(l[i+1:], 10, 64)
return ts, err == nil && ts > 1e15
}

242
cmd/nmon-devtools/seed.go Normal file
View File

@@ -0,0 +1,242 @@
package main
import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
"github.com/v10networks/nmon/internal/profile"
"github.com/v10networks/nmon/internal/simagent"
)
// seedConfig is the command line of `devtools seed-demo`.
type seedConfig struct {
api, hotURL string
days int
step time.Duration
countersRec string
envOut string
profilesDir string
workers int
hc *http.Client
}
// seeded is one device after it was added to nmon.
type seeded struct {
demoDevice
slug string
agent *simagent.Agent
ip string
ifaces []seedIface
}
type seedIface struct {
ID string `json:"id"`
Name string `json:"ifname"`
HC bool `json:"has_hc_counters"`
Excluded bool `json:"excluded"`
Unreliable bool `json:"unreliable"`
}
func (i seedIface) graphable() bool { return i.HC && !i.Excluded && !i.Unreliable }
func seedDemoCmd(ctx context.Context, args []string) error {
cfg := seedConfig{hc: &http.Client{Timeout: 3 * time.Minute}}
fs := devFlagSet("seed-demo")
fs.StringVar(&cfg.api, "api", "http://127.0.0.1:8443", "base URL of the running nmon (started with --insecure-http)")
fs.StringVar(&cfg.hotURL, "hot-url", "", "base URL of the hot VictoriaMetrics, for the back-fill (required)")
fs.IntVar(&cfg.days, "days", 31, "days of history to back-fill")
fs.DurationVar(&cfg.step, "step", 60*time.Second, "back-fill sample spacing")
fs.StringVar(&cfg.countersRec, "counters-rec", "testdata/snmprec/counters.snmprec", "the counters fixture recording")
fs.StringVar(&cfg.envOut, "env-out", "", "write NMON_DEVICE=<counters slug> and NMON_DEVICE_48=<48-port slug> to this file when ready")
fs.StringVar(&cfg.profilesDir, "profiles-dir", "", "operator profiles directory (shipped profiles are always loaded)")
fs.IntVar(&cfg.workers, "workers", 6, "parallel back-fill workers")
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
if cfg.hotURL == "" {
return errors.New("--hot-url is required")
}
cfg.api, cfg.hotURL = strings.TrimRight(cfg.api, "/"), strings.TrimRight(cfg.hotURL, "/")
return cfg.run(ctx)
}
func (c *seedConfig) run(ctx context.Context) error {
set, err := profile.Load(c.profilesDir)
if err != nil {
return err
}
fleet, err := buildFleet(set, c.countersRec)
if err != nil {
return err
}
var pool simagent.Pool
defer func() { _ = pool.Close() }() // process is exiting
added, err := c.addAll(ctx, &pool, fleet)
if err != nil {
return err
}
if err := c.backfillAll(ctx, added); err != nil {
return err
}
if err := c.writeEnv(added); err != nil {
return err
}
fmt.Println("seed-demo ready: agents keep answering until interrupted")
<-ctx.Done()
return nil
}
// addAll starts one agent per device on [::1] with its own port and adds the
// device through the API with that port.
func (c *seedConfig) addAll(ctx context.Context, pool *simagent.Pool, fleet []demoDevice) ([]seeded, error) {
var out []seeded
for i, d := range fleet {
cfg := simagent.Config{Addr: "[::1]:0", Recording: d.rec, Community: d.community}
if err := pool.Start(ctx, []simagent.Config{cfg}); err != nil {
return nil, fmt.Errorf("%s: %w", d.name, err)
}
agent := pool.Agents()[i]
s, err := c.addDevice(ctx, d, "::1", int(agent.Addr().Port()))
s.agent = agent
if err != nil {
return nil, fmt.Errorf("add %s: %w", d.name, err)
}
fmt.Printf("added %-12s %-10s %s (%d interfaces)\n", d.name, d.kind, s.slug, len(s.ifaces))
out = append(out, s)
}
return out, nil
}
func (c *seedConfig) addDevice(ctx context.Context, d demoDevice, ip string, port int) (seeded, error) {
version := d.version
if version == "" {
version = "v2c"
}
body := map[string]any{
"name": d.name, "addresses": []string{ip}, "site": d.site, "role": d.role,
"snmp": map[string]any{"version": version, "community": d.community, "port": port},
}
var created struct {
Slug string `json:"slug"`
}
if err := httpJSON(ctx, c.hc, http.MethodPost, c.api+"/api/v1/devices", body, &created); err != nil {
return seeded{}, err
}
s := seeded{demoDevice: d, slug: created.Slug, ip: ip}
var got struct {
Interfaces []seedIface `json:"interfaces"`
}
if err := httpJSON(ctx, c.hc, http.MethodGet, c.api+"/api/v1/devices/"+url.PathEscape(created.Slug), nil, &got); err != nil {
return seeded{}, err
}
if len(got.Interfaces) == 0 {
return seeded{}, errors.New("discovery found no interfaces")
}
s.ifaces = got.Interfaces
return s, nil
}
// backfillAll waits for each device's first live samples, then writes history
// for every graphable interface in parallel.
func (c *seedConfig) backfillAll(ctx context.Context, devs []seeded) error {
type job struct{ dev, iface string }
var jobs []job
probe := &backfillConfig{hotURL: c.hotURL, hc: c.hc}
for _, d := range devs {
for _, i := range d.ifaces {
if d.hc && i.graphable() {
jobs = append(jobs, job{d.slug, i.ID})
}
}
if err := c.waitLive(ctx, probe, d); err != nil {
return err
}
if d.kind == "down" {
_ = d.agent.Close() // goes silent now, so it reads as down by the time the screenshots are taken
}
}
ch := make(chan job)
errs := make(chan error, len(jobs))
var wg sync.WaitGroup
for w := 0; w < max(1, c.workers); w++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := range ch {
cfg := &backfillConfig{hotURL: c.hotURL, device: j.dev, days: c.days, step: c.step, pattern: patternDiurnal,
seed: 1, hc: c.hc, logf: func(string, ...any) {}}
errs <- cfg.backfillIface(ctx, j.iface)
}
}()
}
for _, j := range jobs {
ch <- j
}
close(ch)
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
return err
}
}
fmt.Printf("back-filled %d interfaces, %d days at %s\n", len(jobs), c.days, c.step)
return nil
}
// waitLive blocks until the hot store holds two live samples of the device's
// first graphable interface, so the back-fill can end where they begin.
func (c *seedConfig) waitLive(ctx context.Context, probe *backfillConfig, d seeded) error {
if !d.hc {
return nil
}
var first string
for _, i := range d.ifaces {
if i.graphable() {
first = i.ID
break
}
}
if first == "" {
return nil
}
deadline := time.Now().Add(3 * time.Minute)
for time.Now().Before(deadline) {
live, err := probe.probe(ctx, "ifHCInOctets", first)
if err == nil && live.ok && live.rate > 0 {
return nil
}
if !sleepCtx(ctx, 2*time.Second) {
return ctx.Err()
}
}
return fmt.Errorf("%s: no live samples in the hot store after 3 minutes (is nmon polling? --poll-interval)", d.slug)
}
func (c *seedConfig) writeEnv(devs []seeded) error {
var b bytes.Buffer
for _, d := range devs {
switch d.kind {
case "counters":
fmt.Fprintf(&b, "NMON_DEVICE=%s\n", d.slug)
case "big":
fmt.Fprintf(&b, "NMON_DEVICE_48=%s\n", d.slug)
case "v1":
fmt.Fprintf(&b, "NMON_DEVICE_32BIT=%s\n", d.slug)
case "down":
fmt.Fprintf(&b, "NMON_DEVICE_DOWN=%s\n", d.slug)
}
}
if c.envOut == "" {
return nil
}
return os.WriteFile(c.envOut, b.Bytes(), 0o600)
}

View File

@@ -0,0 +1,84 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"github.com/v10networks/nmon/internal/metrics"
)
// parseExport reads VictoriaMetrics' /api/v1/export (JSON lines) and returns
// the earliest sample of the first series.
func parseExport(raw []byte) (liveSeries, error) {
line, _, _ := bytes.Cut(bytes.TrimSpace(raw), []byte("\n"))
if len(line) == 0 {
return liveSeries{}, nil
}
var s struct {
Values []float64 `json:"values"`
Timestamps []int64 `json:"timestamps"`
}
if err := json.Unmarshal(line, &s); err != nil {
return liveSeries{}, fmt.Errorf("decode export: %w", err)
}
if len(s.Values) == 0 || len(s.Values) != len(s.Timestamps) {
return liveSeries{}, nil
}
out := liveSeries{ts: time.UnixMilli(s.Timestamps[0]).UTC(), val: s.Values[0], ok: true}
if len(s.Values) > 1 && s.Timestamps[1] > s.Timestamps[0] {
out.rate = (s.Values[1] - s.Values[0]) / (float64(s.Timestamps[1]-s.Timestamps[0]) / 1000)
}
return out, nil
}
func (c *backfillConfig) get(ctx context.Context, u string, dst io.Writer) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return err
}
resp, err := c.hc.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }() // read to the end below
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
return fmt.Errorf("GET %s: %s: %s", u, resp.Status, b)
}
_, err = io.Copy(dst, resp.Body)
return err
}
// importBatch is how many samples go in one POST (about 5 MB of text).
const importBatch = 50_000
// send writes samples through /api/v1/import/prometheus in batches.
func (c *backfillConfig) send(ctx context.Context, samples []metrics.Sample) error {
for len(samples) > 0 {
n := min(len(samples), importBatch)
body := make([]byte, 0, n*110)
for _, s := range samples[:n] {
body = metrics.AppendPromText(body, s)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.hotURL+"/api/v1/import/prometheus", bytes.NewReader(body))
if err != nil {
return err
}
resp, err := c.hc.Do(req)
if err != nil {
return err
}
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
_ = resp.Body.Close() // reply already read
if resp.StatusCode/100 != 2 {
return fmt.Errorf("import: %s: %s", resp.Status, msg)
}
samples = samples[n:]
}
return nil
}

View File

@@ -0,0 +1,219 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/v10networks/nmon/internal/deps"
"github.com/v10networks/nmon/internal/fixturegen"
"github.com/v10networks/nmon/internal/simagent"
)
// Topology commands for the M4 gate: one simulated SNMP agent per device of a
// topology file (deps.Topology), the devices added to a running nmon with their
// parents, and a webhook receiver that records what nmon sends.
const agentCommunity = "public"
// topologyPort is the UDP port of the i-th device of a topology.
func topologyPort(base, i int) int { return base + i }
// topologyAgentsCmd runs one agent per device on [::1]:port-base+index. The
// agent of device X is down while the file <control-dir>/stop-X exists, so a
// shell script can take a device away and bring it back on the same port.
func topologyAgentsCmd(ctx context.Context, args []string) error {
fs := devFlagSet("topology-agents")
file := fs.String("topology", "", "topology file (testdata/topologies/*.yaml)")
base := fs.Int("port-base", 0, "UDP port of the first device; device i uses port-base+i")
ctl := fs.String("control-dir", "", "directory watched for stop-<slug> files")
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
if *file == "" || *base == 0 || *ctl == "" {
return fmt.Errorf("topology-agents needs --topology, --port-base and --control-dir")
}
topo, err := deps.LoadTopology(*file)
if err != nil {
return err
}
f := &agentFleet{topo: topo, base: *base, ctl: *ctl, agents: map[string]*simagent.Agent{}}
defer f.closeAll()
for {
if err := f.reconcile(ctx); err != nil {
return err
}
if !f.announced {
f.announced = true
fmt.Printf("agents ready: %d devices on [::1]:%d-%d\n", len(topo.Devices), *base, topologyPort(*base, len(topo.Devices)-1))
}
if !sleepCtx(ctx, 200*time.Millisecond) {
return nil
}
}
}
type agentFleet struct {
topo *deps.Topology
base int
ctl string
agents map[string]*simagent.Agent
announced bool
}
// reconcile starts the agents whose stop file is gone and stops those whose stop file appeared.
func (f *agentFleet) reconcile(ctx context.Context) error {
for i, d := range f.topo.Devices {
_, stopped := os.Stat(filepath.Join(f.ctl, "stop-"+d.Slug))
want := stopped != nil // no stop file: the agent should run
a := f.agents[d.Slug]
switch {
case want && a == nil:
rec := fixturegen.Generic(fixturegen.Options{Ports: 4, HCCounters: true, Counters: true, SysName: d.Slug})
cfg := simagent.Config{Addr: fmt.Sprintf("[::1]:%d", topologyPort(f.base, i)), Recording: rec, Community: agentCommunity}
ag, err := simagent.Start(ctx, cfg)
if err != nil {
return fmt.Errorf("agent %s: %w", d.Slug, err)
}
f.agents[d.Slug] = ag
case !want && a != nil:
_ = a.Close() // best-effort: the port is released either way
delete(f.agents, d.Slug)
}
}
return nil
}
func (f *agentFleet) closeAll() {
for _, a := range f.agents {
_ = a.Close() // best-effort at exit
}
}
// topologyAddCmd adds every device of a topology to a running nmon, pointing at
// its agent, then sets the parents by slug.
func topologyAddCmd(ctx context.Context, args []string) error {
fs := devFlagSet("topology-add")
api := fs.String("api", "", "nmon API base URL")
file := fs.String("topology", "", "topology file")
base := fs.Int("port-base", 0, "UDP port of the first device's agent")
mapOut := fs.String("map-out", "", "file that receives one \"<nmon slug> <topology name>\" line per device")
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
if *api == "" || *file == "" || *base == 0 {
return fmt.Errorf("topology-add needs --api, --topology and --port-base")
}
topo, err := deps.LoadTopology(*file)
if err != nil {
return err
}
hc := &http.Client{Timeout: 90 * time.Second}
slugOf := map[string]string{} // topology name -> the slug nmon generated
for i, d := range topo.Devices {
body := map[string]any{
"name": d.Slug, "addresses": []string{"::1"}, "role": d.Role, "kind": d.Kind,
"snmp": map[string]any{"version": "v2c", "community": agentCommunity, "port": topologyPort(*base, i)},
}
var got struct {
Slug string `json:"slug"`
}
if err := httpJSON(ctx, hc, http.MethodPost, *api+"/api/v1/devices", body, &got); err != nil {
return fmt.Errorf("add %s: %w", d.Slug, err)
}
slugOf[d.Slug] = got.Slug // nmon appends a random suffix to the slug it derives from the name
}
for _, d := range topo.Devices {
if len(d.Parents) == 0 {
continue
}
parents := make([]string, len(d.Parents))
for i, p := range d.Parents {
parents[i] = slugOf[p]
}
err := httpJSON(ctx, hc, http.MethodPut, *api+"/api/v1/devices/"+url.PathEscape(slugOf[d.Slug]), map[string]any{"parents": parents}, nil)
if err != nil {
return fmt.Errorf("parents of %s: %w", d.Slug, err)
}
}
fmt.Printf("topology-add ready: %d devices\n", len(topo.Devices))
return writeSlugMap(*mapOut, topo, slugOf)
}
// writeSlugMap records which nmon slug belongs to which topology name, for gate-dump --map and the gate script.
func writeSlugMap(path string, topo *deps.Topology, slugOf map[string]string) error {
if path == "" {
return nil
}
var b strings.Builder
for _, d := range topo.Devices {
fmt.Fprintf(&b, "%s %s\n", slugOf[d.Slug], d.Slug)
}
return os.WriteFile(path, []byte(b.String()), 0o600)
}
// sinkLine is one recorded request.
type sinkLine struct {
At string `json:"at"`
Path string `json:"path"`
Body json.RawMessage `json:"body"`
}
// webhookSinkCmd records every POST body as one JSON line in --out, answers 200,
// and writes the bound port to --port-file once it listens.
func webhookSinkCmd(ctx context.Context, args []string) error {
fs := devFlagSet("webhook-sink")
listen := fs.String("listen", "[::1]:0", "listen address")
out := fs.String("out", "", "file that receives one JSON line per request")
portFile := fs.String("port-file", "", "file that receives the bound port")
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
if *out == "" {
return fmt.Errorf("webhook-sink needs --out")
}
f, err := os.OpenFile(*out, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return err
}
defer func() { _ = f.Close() }() // lines are written whole; nothing is buffered
ln, err := net.Listen("tcp", *listen)
if err != nil {
return err
}
if *portFile != "" {
port := ln.Addr().(*net.TCPAddr).Port
if err := os.WriteFile(*portFile, []byte(fmt.Sprint(port)), 0o600); err != nil {
return err
}
}
var mu sync.Mutex
srv := &http.Server{ReadHeaderTimeout: 10 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := io.ReadAll(io.LimitReader(r.Body, 1<<20))
raw := json.RawMessage(b)
if !json.Valid(b) {
raw, _ = json.Marshal(string(b))
}
line, _ := json.Marshal(sinkLine{At: time.Now().UTC().Format(time.RFC3339Nano), Path: r.URL.Path, Body: raw})
mu.Lock()
_, _ = f.Write(append(line, '\n'))
mu.Unlock()
w.WriteHeader(http.StatusOK)
})}
go func() { <-ctx.Done(); _ = srv.Close() }()
fmt.Printf("webhook-sink listening on %s\n", ln.Addr())
if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}

25
cmd/nmon/openapi.go Normal file
View File

@@ -0,0 +1,25 @@
package main
import (
"context"
"fmt"
"os"
"github.com/v10networks/nmon/internal/app"
)
func init() {
register("openapi", "print the generated OpenAPI 3.1 document (nmon openapi > openapi.json)", openapiCmd)
}
func openapiCmd(_ context.Context, args []string) error {
if len(args) > 0 {
return fmt.Errorf("openapi takes no arguments, got %q", args[0])
}
spec, err := app.OpenAPISpec()
if err != nil {
return fmt.Errorf("generate spec: %w", err)
}
_, err = fmt.Fprintf(os.Stdout, "%s\n", spec)
return err
}

View File

@@ -11,9 +11,8 @@ import (
"github.com/gosnmp/gosnmp"
"github.com/v10networks/nmon/internal/fixturegen"
"github.com/v10networks/nmon/internal/simagent"
"github.com/v10networks/nmon/internal/snmp"
"github.com/v10networks/nmon/internal/snmprec"
)
// defaultRecordSubtrees is what `profile record` walks when --subtree is not given:
@@ -130,7 +129,7 @@ func recordDevice(ctx context.Context, o recordOpts, w io.Writer) (int, error) {
return 0, err
}
defer func() { _ = s.Close() }() // read-only session
var recs []simagent.Record
var recs []snmprec.Record
for _, root := range o.Subtrees {
err := s.Walk(ctx, root, o.MaxReps, func(vb snmp.Varbind) error {
if vb.Exception() {
@@ -146,19 +145,19 @@ func recordDevice(ctx context.Context, o recordOpts, w io.Writer) (int, error) {
return 0, fmt.Errorf("walk %s: %w", root, err)
}
}
rec := simagent.NewRecording(recs)
rec := snmprec.NewRecording(recs)
if rec.Len() == 0 {
return 0, fmt.Errorf("the device returned nothing under %v", o.Subtrees)
}
if err := fixturegen.WriteSnmprec(w, rec); err != nil {
if err := snmprec.Write(w, rec); err != nil {
return 0, err
}
return rec.Len(), nil
}
// redact turns a varbind into a recording row, replacing secrets with REDACTED.
func redact(vb snmp.Varbind, community string) simagent.Record {
r := simagent.Record{OID: vb.OID, Type: vb.Type, Value: vb.Value}
func redact(vb snmp.Varbind, community string) snmprec.Record {
r := snmprec.Record{OID: vb.OID, Type: vb.Type, Value: vb.Value}
if redactedOIDs[vb.OID] {
r.Type, r.Value = gosnmp.OctetString, []byte(redacted)
return r

View File

@@ -14,6 +14,7 @@ import (
"github.com/v10networks/nmon/internal/profile"
"github.com/v10networks/nmon/internal/simagent"
"github.com/v10networks/nmon/internal/snmprec"
)
func bundledMIBs(t *testing.T) string {
@@ -121,7 +122,7 @@ const recordSample = `1.3.6.1.2.1.1.1.0|4|Test device
`
func TestProfileRecordRedacts(t *testing.T) {
rec, err := simagent.ParseRecording(strings.NewReader(recordSample))
rec, err := snmprec.Parse(strings.NewReader(recordSample))
if err != nil {
t.Fatal(err)
}
@@ -148,7 +149,7 @@ func TestProfileRecordRedacts(t *testing.T) {
if n := strings.Count(s, "REDACTED"); n != 4 {
t.Errorf("v%s: want 4 REDACTED values, got %d:\n%s", ver, n, s)
}
got, err := simagent.ParseRecording(strings.NewReader(s))
got, err := snmprec.Parse(strings.NewReader(s))
if err != nil {
t.Fatalf("v%s: output does not parse: %v", ver, err)
}
@@ -188,7 +189,7 @@ func TestProfileTestArgs(t *testing.T) {
}
func TestProfileTestPrintsMetrics(t *testing.T) {
rec, err := simagent.LoadRecording("../../testdata/snmprec/juniper-junos.snmprec")
rec, err := snmprec.Load("../../testdata/snmprec/juniper-junos.snmprec")
if err != nil {
t.Fatal(err)
}
@@ -222,3 +223,20 @@ func TestNmonDoesNotImportLibreNMS(t *testing.T) {
}
}
}
// Dev and test machinery stays out of the product binary: nmon-devtools and tests own the simulator,
// the fixture generator and the fixtures. `nmon profile record` needs only internal/snmprec.
func TestNmonDoesNotLinkDevMachinery(t *testing.T) {
out, err := exec.Command("go", "list", "-deps", ".").CombinedOutput()
if err != nil {
t.Skipf("go list unavailable: %v\n%s", err, out)
}
banned := []string{"internal/simagent", "internal/fixturegen", "internal/fixtures", "internal/librenms", "internal/testutil"}
for _, pkg := range strings.Fields(string(out)) {
for _, b := range banned {
if strings.HasSuffix(pkg, "/"+b) {
t.Errorf("cmd/nmon depends on %s", pkg)
}
}
}
}

108
cmd/nmon/rollup.go Normal file
View File

@@ -0,0 +1,108 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"github.com/v10networks/nmon/internal/cliout"
"github.com/v10networks/nmon/internal/rollup"
)
func init() {
register("rollup", "repair rollup windows (asks a running nmon)", rollupCmd)
}
const rollupUsage = "usage: nmon rollup repair <from> <to> [--api URL] [--token TOKEN]\n\n" +
"Re-runs every rollup window overlapping [from, to) on the running nmon, writing a new version of\n" +
"each. The range must be inside hot retention (default 30 days). Times are RFC 3339 (2026-10-20T08:00:00Z)\n" +
"or a UTC date (2026-10-20); to is exclusive."
func rollupCmd(ctx context.Context, args []string) error {
if len(args) == 0 || args[0] != "repair" {
return fmt.Errorf("%s", rollupUsage)
}
fs := profileFlagSet("rollup repair", "rollup repair <from> <to> [--api URL]")
var api, token string
var timeout time.Duration
fs.StringVar(&api, "api", "http://[::1]:8443", "base URL of the running nmon")
fs.StringVar(&token, "token", os.Getenv("NMON_TOKEN"), "admin API token (default $NMON_TOKEN)")
fs.DurationVar(&timeout, "timeout", 30*time.Minute, "how long to wait for the repair")
pos, err := parseInterspersed(fs, args[1:])
if err != nil {
return flagErr(err)
}
if len(pos) != 2 {
return fmt.Errorf("%s", rollupUsage)
}
from, err := parseRepairTime(pos[0])
if err != nil {
return err
}
to, err := parseRepairTime(pos[1])
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
rep, err := requestRepair(ctx, api, token, rollup.RepairRequest{From: from, To: to})
if err != nil {
return err
}
w := cliout.New(os.Stdout)
w.Printf("repaired %s to %s: %d five-minute windows, %d hourly windows rewritten\n",
rep.From.Format(time.RFC3339), rep.To.Format(time.RFC3339), rep.FiveMinutes, rep.Hours)
return w.Err()
}
func parseRepairTime(s string) (time.Time, error) {
for _, layout := range []string{time.RFC3339, "2006-01-02T15:04", "2006-01-02"} {
if t, err := time.ParseInLocation(layout, s, time.UTC); err == nil {
return t.UTC(), nil
}
}
return time.Time{}, fmt.Errorf("cannot parse %q as RFC 3339 or YYYY-MM-DD", s)
}
// requestRepair calls POST /api/v1/rollup/repair, which internal/app mounts on
// rollup.Job.Repair.
func requestRepair(ctx context.Context, api, token string, rr rollup.RepairRequest) (*rollup.RepairReport, error) {
body, err := json.Marshal(rr)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(api, "/")+"/api/v1/rollup/repair", bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot reach nmon at %s (is it running? use --api): %w", api, err)
}
defer func() { _ = resp.Body.Close() }() // body fully read below; a close error changes nothing
out, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if resp.StatusCode != http.StatusOK {
var p struct{ Title, Detail string }
_ = json.Unmarshal(out, &p)
msg := p.Detail
if msg == "" {
msg = p.Title
}
return nil, fmt.Errorf("repair refused: %d %s", resp.StatusCode, msg)
}
var rep rollup.RepairReport
if err := json.Unmarshal(out, &rep); err != nil {
return nil, err
}
return &rep, nil
}

48
cmd/nmon/rollup_test.go Normal file
View File

@@ -0,0 +1,48 @@
package main
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/v10networks/nmon/internal/rollup"
)
func TestRollupRepairSendsRangeAndTokenToRunningNmon(t *testing.T) {
var got rollup.RepairRequest
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/rollup/repair" || r.Header.Get("Authorization") != "Bearer tok" {
http.Error(w, "bad request", http.StatusBadRequest)
return
}
_ = json.NewDecoder(r.Body).Decode(&got)
_ = json.NewEncoder(w).Encode(rollup.RepairReport{From: got.From, To: got.To, FiveMinutes: 12, Hours: 1})
}))
defer srv.Close()
from, to := time.Date(2026, 10, 20, 8, 0, 0, 0, time.UTC), time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)
rep, err := requestRepair(context.Background(), srv.URL, "tok", rollup.RepairRequest{From: from, To: to})
if err != nil || rep.FiveMinutes != 12 || !got.From.Equal(from) || !got.To.Equal(to) {
t.Fatalf("rep=%+v got=%+v err=%v", rep, got, err)
}
if _, err := requestRepair(context.Background(), srv.URL, "wrong", rollup.RepairRequest{}); err == nil {
t.Error("a refused repair must be an error")
}
}
func TestParseRepairTimeAcceptsDatesAndRFC3339(t *testing.T) {
for in, want := range map[string]time.Time{
"2026-10-20": time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC),
"2026-10-20T08:30": time.Date(2026, 10, 20, 8, 30, 0, 0, time.UTC),
"2026-10-20T08:00:00Z": time.Date(2026, 10, 20, 8, 0, 0, 0, time.UTC),
} {
if got, err := parseRepairTime(in); err != nil || !got.Equal(want) {
t.Errorf("%s: %v %v", in, got, err)
}
}
if _, err := parseRepairTime("yesterday"); err == nil {
t.Error("want error")
}
}

View File

@@ -62,6 +62,10 @@ func serve(ctx context.Context, args []string) error {
fs.StringVar(&f.TrapListen, "trap-listen", app.DefaultTrapListen,
"UDP address for SNMP traps; empty disables the receiver "+
"(port 162 needs CAP_NET_BIND_SERVICE; without it traps are off and polling continues)")
fs.StringVar(&f.GraphiteListen, "graphite-listen", app.DefaultGraphiteListen,
"TCP address for Graphite plaintext push (first line \"token <secret>\"); empty disables the listener")
fs.StringVar(&f.ICMP, "icmp", config.ICMPAuto,
"ICMP probing: auto, or off for hosts where ICMP is not permitted (SNMP alone then decides if a device is down)")
fs.IntVar(&f.MaxInflight, "max-inflight", 512, "most SNMP polls in flight at once")
fs.DurationVar(&f.PollInterval, "poll-interval", 0, "dev: SNMP poll interval (default 60s)")
fs.IntVar(&f.SNMPPort, "snmp-port", 0, "dev: UDP port of every device's agent (default 161)")

View File

@@ -14,3 +14,24 @@ conscious, reviewed exception.
| 7 | `import-librenms` reads `includes/definitions/*.yaml` and `includes/definitions/discovery/*.yaml` | Reads `resources/definitions/os_detection/*.yaml` and `resources/definitions/os_discovery/*.yaml`, with the old paths as a fallback. | LibreNMS moved them; the spec's paths no longer exist upstream. | M2 |
| 8 | Real recordings replace synthetic fixtures when available | LibreNMS's `tests/snmpsim/*.snmprec` (2,054 real device recordings) are used by optional tests when `NMON_LIBRENMS_DIR` points at a checkout. They are never copied into this repo. | Real-device coverage without vendoring GPL test data into an Apache-2.0 repo. | M2 |
| 9 | `nmon profile import-librenms <repo-path>` is an nmon subcommand | A separate binary, `nmon-librenms-import` (`cmd/nmon-librenms-import`), run offline before nmon. It writes draft profiles that the operator reviews and copies into `<data-dir>/profiles/`. `nmon` neither links nor exposes it. | User decision (2026-10-22). It keeps the importer out of the product binary and makes "the converter never runs at serve time" structural. | M2 |
| 10 | `scripts/openapi-diff` fails on an incompatible change (M3 milestone) | The differ is the Go package `internal/openapidiff` (unit-tested, run with `go run ./internal/openapidiff/cmd/openapidiff`), driven by `scripts/openapi-check.sh`, which `make gate-m3` and the CI `openapi` job call. | A Go package is testable and linted like the rest of the code; the script only fetches the spec at the last `m*`/`v*` tag. No new dependency. | M3 |
| 11 | Built assets are embedded from `web/dist` into `internal/webui` | `make web` copies `web/dist` to `internal/webui/dist` (git-ignored, `.gitkeep` tracked); a committed `internal/webui/placeholder/index.html` ("UI not built") is served when `dist` has no `index.html`. | A tracked `dist/index.html` would be overwritten by every build and dirty the tree, and a built UI must never be committed. `go build` still works without node. | M3 |
| 12 | (not specified) | Developer tools (`backfill`, `seed-demo`) are a separate executable, `cmd/nmon-devtools` (`make devtools` builds `nmon-devtools`), never shipped. It talks to a running nmon over its HTTP API and to the hot store via `--hot-url`. `nmon` has no `devtools` command; depguard and a `go list -deps` test keep `cmd/nmon` from linking `internal/simagent`, `internal/fixturegen`, `internal/fixtures` or `internal/testutil`; `nmon profile record` uses only the `.snmprec` format in `internal/snmprec`. | The gate needs 30 days of history without waiting 30 days, and live screenshots need a simulated network. See `docs/dev.md`. | M3 |
| 13 | (not specified) `/docs` API reference | `/docs` serves a vendored, pinned swagger-ui-dist 5.17.14 (Apache-2.0, about 1.6 MB) through `embed.FS`. | huma's built-in renderers load their assets from a CDN, which an offline monitor cannot rely on. | M3 |
| 14 | M2 defined provisional `poller/health` and `identity-reviews` responses | Both were reshaped in M3 to the reviewed UI contract, before any `openapi.json` existed, so the route freeze starts at `m3`, the first tag with a checked-in spec. | The M2 shapes had no consumer. `scripts/openapi-check.sh` passes against `m2` with a note and diffs for real from `m3`. | M3 |
| 15 | A single-signal failure (ICMP only or SNMP only) is a warning | The warning is held as pending for 5 min before it notifies, and is closed silently if the device goes down meanwhile. | Otherwise a real outage whose second signal fails seconds later sends a spurious warning first. | M4 |
| 16 | An unsupported probe is unknown, never failed | When ICMP is unsupported (host can't open unprivileged ICMP, or per device), SNMP alone decides down, as with `icmp_blocked`, and a one-time `icmp_unsupported` device warning is raised. | Otherwise nmon could never declare a device down on such a host, and nobody would notice. | M4 |
| 17 | M6 gate: "the 2,000-device loadtest cycle finishes under 40 s"; the loadtest acceptance lines (200k interfaces in 40 s, memory stable over 30 min, peak RSS/disk/ingest/latency report, backup during load) | **Skipped** (approved by the owner, 2026-10-22: "skip the device load test… put this in prod asap"). The M6 gate checks only that a viewer gets 403 on PUT. Backup/restore is verified on a normal-size dataset instead of under load. Scale is unproven beyond the fixture and gate sizes (24 devices). | Time to production. | M6 |
| 18 | Push token format not specified | The secret is `nmp_<id>_<base32 of 160 random bits>`; the row stores a salted SHA-256 (`<salt>$<hash>`). | The id selects the row, so authentication is one primary-key lookup and never scans or compares other tokens. | M5 |
| 19 | `rollup_windows` status is `done` or `partial` | A third status, `pending`, is written before the long-store write begins. | A crash between the write and the commit still tells the next run which version number is taken, so it writes `v+1`. | M5 |
| 20 | `v` is a version number on rollup series | `v` is on `nmon_if_rate_bps_5m`, `_avg_1h`, `_max_1h` and the gauge series, encoded `YYYYMM*1000+n` (UTC month of the window, then the version). | Numeric, so MetricsQL can compare versions; the month prefix lets retention drop whole months of 5 m series. Readers use `internal/longq.LatestVersion`. | M5 |
| 21 | Gauge rollups: "a 5 m average, plus a 1 h avg, max and min" | Series are named `<name>_avg_5m`, `<name>_avg_1h`, `<name>_max_1h`, `<name>_min_1h`, with the source labels plus `v`. | The suffix keeps the source name searchable and cannot collide with a polled metric (such names are not rolled up). | M5 |
| 22 | 5 m series are deleted past `retention.fiveminute` | Whole calendar months are deleted with `delete_series` once they ended before `now - retention.fiveminute`, so 5 m data lives 4 to 5 months. | Open-source VictoriaMetrics has one retention per process and no time-ranged delete; the `v` prefix makes each 5 m series one month. | M5 |
| 23 | Rollup writes go through the write queue | Rollup long writes bypass the queue: a synchronous, acknowledged import, then `force_flush`, then a read-back count. | The queue is fire-and-forget (it can drop after 60 s); a window may be recorded done only once the store has it. | M5 |
| 24 | Unreliable (32-bit) interfaces are excluded from p95 and group sums | They are also never rolled up, and a graph of one returns no series. | A rate from a wrapping counter is wrong whichever tier stores it. | M5 |
| 25 | `nmon_group_p95_bps{group, dir, month, members_hash}` | Group (and interface) p95 records also carry a `coverage` label. | A figure below 90% coverage must never be shown without it. | M5 |
| 26 | Check samples labelled by device | `nmon_check_up`, `nmon_check_latency_seconds` and `nmon_tls_expiry_seconds` also carry `check` (the check id). | A device can have several checks. | M5 |
| 27 | Proxmox nodes are devices | A node is the existing SNMP device with that name (or sysName), else a new `check_only` device; its health is the status the API reports. Guests are `vm`/`container` devices keyed by `pve:<cluster>/<vmid>`. | `cluster/resources` has no addresses, so nothing but the name can match a node; a missing SNMP device must not orphan its guests. | M5 |
| 28 | Pushed series carry only `device` | Pushed series also carry `push="1"` (senders cannot set it). | The device page lists what a push sender wrote without guessing from metric names. | M5 |
| 29 | Ingest caps, late p95 months and a stalled rollup "raise an alert" | They are rows of the alerts table with rules `ingest_<kind>`, `p95_late`, `rollup_stalled` and no device, kept open until cleared (ingest alarms close 20 min after the last trip). | They are not metric rules and have no device; the metric-rule evaluator leaves these rule names alone. | M5 |
| 30 | Proxmox cluster config could carry the token secret | Only `secret_name` exists; the secret is stored encrypted via `PUT /api/v1/secrets/{name}` and cannot be read back. | `config.yaml` is plain text. | M5 |

50
docs/dev.md Normal file
View File

@@ -0,0 +1,50 @@
# Developer notes
## Building the UI into the binary
`make build` runs `make web` first: `npm run build` in `web/`, then `web/dist` is copied to
`internal/webui/dist`, which `//go:embed` packs into the binary. `internal/webui/dist` is git-ignored
(only `.gitkeep` is tracked), so a build never dirties the tree. A plain `go build` without `make web`
embeds a one-line "UI not built" page from `internal/webui/placeholder/`.
nmon serves the UI at `/`. Unknown paths fall back to `index.html` for the client-side router, except
`/api/`, `/docs`, `/openapi*`, `/metrics` and `/schemas`, which belong to the API. Files under `/assets/`
are content-hashed and sent `immutable`; `index.html` is `no-cache`; text assets are served gzipped.
## OpenAPI compatibility
`scripts/openapi-check.sh` compares the current spec (`nmon openapi`, else `openapi.json`) with
`openapi.json` at the latest `m*`/`v*` tag and fails on a removed route or method, a removed response
field, a changed field type, a new required parameter or request field, or a removed enum value. A tag
without `openapi.json` passes with a note. `OPENAPI_NEW=file.json` compares that file instead and
`OPENAPI_BASE_TAG=m3` picks the base. The comparison is `internal/openapidiff`.
## `nmon-devtools` (separate dev binary)
Developer helpers in their own executable, `make devtools` -> `nmon-devtools`; the shipped `nmon` does not contain them and never links the simulator. They need a running `nmon serve --insecure-http`.
**backfill** writes synthetic history straight into the hot store so graphs have 30 days of data without
waiting 30 days:
nmon-devtools backfill --hot-url http://127.0.0.1:PORT --device SLUG --iface ULID [--iface ULID...] \
[--days 31] [--step 60s] [--pattern diurnal|flat] [--mean-mbps N]
Per interface it writes `ifHCInOctets`, `ifHCOutOctets` (monotonic, diurnal rate with noise and bursts) and
`ifOperStatus`, ending one step before the first live sample. When live samples exist the history is shifted
so its last value joins the live counter with no reset (this needs a live counter larger than the history,
about 3e13 bytes for 31 days at 100 Mbit/s; otherwise a warning says the join will look like a reset).
The hot URL is in the serve log (`metric stores up hot=...`). Samples older than hot retention are dropped
by the store.
**seed-demo** builds a small simulated network and back-fills it, for the gate and for screenshots:
nmon serve --data-dir D --insecure-http --poll-interval 10s ...
nmon-devtools seed-demo --api http://127.0.0.1:8443 --hot-url http://127.0.0.1:PORT \
[--env-out env] [--counters-rec testdata/snmprec/counters.snmprec]
It starts the simulated SNMP agents in-process on `[::1]`, each on its own port (passed to the API as `snmp.port`), adds the devices through the API (a 48-port switch, access switches, routers, a firewall, a UPS, the counters fixture, a v1 device with 32-bit counters, and one whose agent is stopped after seeding so it reads as down), waits for live polls, back-fills 31 days, writes `NMON_DEVICE`, `NMON_DEVICE_48`, `NMON_DEVICE_32BIT` and `NMON_DEVICE_DOWN` to `--env-out`, and keeps the agents answering until interrupted. The simulated 64-bit counters start at 1e14 so history fits below them.
## Gates
`make gate-m3` runs everything above end to end and writes live screenshots to `docs/screenshots/M3/`.
It needs Playwright's chromium: `cd web && npx playwright install chromium`.

View File

@@ -33,7 +33,20 @@ API and UI: openapi.json generated with the TS client and the incompatible-diff
## Gate log
_(filled in by the supervisor when the gate passes)_
**Passed 2026-10-22**, run by the supervisor (Opus).
| Check | Result |
|---|---|
| `make lint` (Go + web) | 0 findings |
| `go test -race ./...`, `-tags 'integration vmembed'`, `test-pg.sh` | all ok |
| `make gate-m1`, `make gate-m2` (with LibreNMS recordings) | PASS |
| `make gate-m3` | PASS: UI embedded in the static binary; SPA fallback and cache headers; 10 simulated devices (`nmon-devtools seed-demo`) with 31 days of back-filled history; port graph renders in and out at 1 h, 24 h, 30 d; 48-port page < 2 s; openapi-check; web client regenerated from the live spec type-checks |
| UI ↔ API contract | `TestGeneratedSpecMatchesUIContract`: the generated `openapi.json` is compatible with the contract the UI was built against |
| Screenshots | `docs/screenshots/M3/` (real data, 18 files) and `docs/screenshots/M3-mock/` (mock data); reviewed by the supervisor and sent to the owner |
**Decisions in M3:** dev tooling is a separate `nmon-devtools` binary (owner's call), never shipped. `/docs` uses vendored Swagger UI. The two M2 provisional routes were reshaped before the route freeze, which starts at `m3`, the first tag with `openapi.json`. Graph stitching joins at now − hot retention; p95 is computed live until M5 materialises it.
**Carried forward:** a demo device that is genuinely Down needs an unreachable address; that comes with M4's reachability work (`device-unresponsive.png` shows Warning/degraded for now, which is correct per policy because ICMP still answers). `nmon profile record` still links the simulator package; to be moved to a shared `internal/snmprec` format package.
## API contract for M3 (written before the WPs so the UI and API are built in parallel)

View File

@@ -44,4 +44,13 @@ Alerting: all *Must* and *Must not* lines. Adversarial: redundant parents failin
## Gate log
_(filled in by the supervisor when the gate passes)_
**Passed 2026-10-23**, run by the supervisor (Opus).
| Check | Result |
|---|---|
| `make lint` (Go + web) | 0 findings |
| `go test -race ./...`, `-tags 'integration vmembed'`, `test-pg.sh` | all ok |
| gates m1, m2 (LibreNMS recordings), m3, m4, m5 in one sequential run | all PASS |
| `make gate-m4` | PASS: redundant topology (2 cores, 2 dual-homed distribution switches, 20 access switches) with `--icmp off`; (a) one core down → exactly one notice, nothing suppressed; (b) **both cores and all 22 children stopped at the same instant → exactly ONE notification** naming both cores and 22 suppressed children, zero per-child notices; (c) restart re-sends nothing; (d) core-1 back → one resolution, dist switches announced with their suppressed children; deliveries logged with redacted payloads; incident IDs stable |
**Decisions in M4:** a dependency hold delays a child's notice while its parents are unsettled (proven over 100 randomised timing seeds), so simultaneous outages group correctly. Unsupported ICMP falls back to SNMP alone and shows an `icmp_unsupported` device warning. Single-signal warnings are held 5 min. `--icmp auto|off`. `alerts?state=open` (open alerts, acked or not) is used on device pages. That fix came from the gate's screenshot step, which caught an acked outage vanishing from its device page.

View File

@@ -45,4 +45,13 @@ Storage and rollups: all lines except the loadtest disk budget (M6). Ingest: all
## Gate log
_(filled in by the supervisor when the gate passes)_
**Passed 2026-10-23**, run by the supervisor (Opus).
| Check | Result |
|---|---|
| `make lint` (Go + web) | 0 findings |
| `go test -race ./...`, `-tags 'integration vmembed'`, `test-pg.sh` | all ok |
| gates m1, m2 (LibreNMS recordings), m3, m4, m5 in one sequential run | all PASS |
| `make gate-m5` | PASS: mock Proxmox synced (node pve1 + guests with hosted_on); `proxmox.influx` replayed via a push token → 93 series under pve1; **2 y graph hits the 1 h tier** (tiers `1h`+`raw`, `stitched_at`, long points exactly the back-filled values); p95 with `source`; group graph is the oriented sum; HTTP check up; ingest negatives (401s, reserved prefix dropped, series-cap alert deduped) |
**Decisions in M5:** see DEVIATIONS rows 18–30 (push token format, rollup `pending` status and `v` encoding, gauge rollup names, month-partitioned 5-min retention, acked synchronous long writes, unreliable interfaces never rolled up, group `coverage` label, check `check` label, Proxmox node rule, `push="1"` label, system alarms, no inline Proxmox secret). One `internal/longq` package now owns the "highest version wins" query; this fixed M3's long-tier graph read.

View File

@@ -1,6 +1,6 @@
# M6: Auth and ops
**Gate:** a viewer gets 403 on PUT, and the 2,000-device loadtest cycle finishes under 40 s.
**Gate:** a viewer gets 403 on PUT. ~~and the 2,000-device loadtest cycle finishes under 40 s~~ (skipped: approved deviation, see `docs/DEVIATIONS.md`; the owner prioritised getting to production).
**Gate command:** `make gate-m6`. It runs the viewer-fuzz test (every state-changing route with a viewer session and a viewer token → 403) and then `nmon loadtest --devices 2000 --ports 100 --duration 30m`, which must report cycle p100 < 40 s, zero deadline misses, write-queue high-water < 50%, and stable RSS. It also prints the report metrics (peak RSS, steady and peak disk, ingest samples/s, p95 device-page latency, write-queue high-water, backlog recovery after a 5 min store outage). Finally it runs backup-restore during load.
## Scope
@@ -32,7 +32,7 @@
| WP1 OIDC, sessions, CSRF, tokens, break-glass, proxy, TLS | `internal/auth/**`, API middleware | M3 API |
| WP2 Backup, restore, key rotate, disk guard, store restart | `internal/backup/**`, `internal/diskguard/**`, `cmd/nmon/{backup,restore,key,store,unlock}.go` | M1 vmsup, store |
| WP3 Config reload, settings API and UI, CSV import, purge | `internal/config/**`, `internal/api/settings*.go`, `internal/importcsv/**`, `web/src/routes/settings*` | M4 deps |
| WP4 Loadtest, viewer fuzz, gate | `internal/loadtest/**`, `cmd/nmon/loadtest.go`, `scripts/gate-m6.sh` | all |
| WP4 Viewer fuzz, backup/restore check, gate (loadtest skipped, approved deviation) | `internal/loadtest/**`, `cmd/nmon/loadtest.go`, `scripts/gate-m6.sh` | all |
## Acceptance lines closed

View File

@@ -14,10 +14,10 @@ its **Gate log** is filled in when the gate passes. A milestone is done only whe
|---|---|---|---|---|
| M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | **passed**, tagged `m1` |
| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | **passed**, tagged `m2` |
| M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | in progress (web UI done on mock data) |
| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | in progress (WP2 rules and WP3 notify done) |
| M5 | Push ingest, rollup, checks | a replayed Proxmox sample shows under its node and a 2 y query hits the 1 h tier | [M5](M5-ingest-rollup-checks.md) | planned |
| M6 | Auth and ops | a viewer gets 403 on PUT and the 2,000-device loadtest cycle finishes under 40 s | [M6](M6-auth-and-ops.md) | planned |
| M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | **passed**, tagged `m3` |
| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | **passed**, tagged `m4` |
| M5 | Push ingest, rollup, checks | a replayed Proxmox sample shows under its node and a 2 y query hits the 1 h tier | [M5](M5-ingest-rollup-checks.md) | **passed**, tagged `m5` |
| M6 | Auth and ops | a viewer gets 403 on PUT (2,000-device loadtest skipped, approved deviation) | [M6](M6-auth-and-ops.md) | in progress |
| M7 | gNMI streaming | after v1; not built in this run | — | out of scope |
## How the work is run
@@ -47,6 +47,8 @@ Departures from the spec are recorded in [`docs/DEVIATIONS.md`](../DEVIATIONS.md
- **Deployment playbook**: install layout (binary, `/usr/share/nmon/mibs` sidecar, systemd unit granting port 162, key file location), TLS/ACME or proxy setup, IdP configuration, backups, and upgrades. Raised by the user on 2026-10-22; deliberately postponed.
- **Web push notifications (v2)**: a `webpush` receiver kind in `internal/notify` (RFC 8291 payload encryption, RFC 8292 VAPID, standard-library crypto), per-user subscriptions (needs M6 users), a service worker in the UI, and push endpoints in the outbound allow-list. Out of v1 per the spec's non-goals ("alerts go to webhooks and email only"). Owner deferred it to v2 on 2026-10-22. Until then, a webhook receiver pointed at ntfy or Gotify gives phone push.
## Review artifacts
- From M3 on, each UI milestone's gate captures Playwright screenshots of every page into `docs/screenshots/<milestone>/` for the user to review.

Binary file not shown.

Before

Width:  |  Height:  |  Size: 136 KiB

After

Width:  |  Height:  |  Size: 141 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 581 KiB

After

Width:  |  Height:  |  Size: 603 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 153 KiB

After

Width:  |  Height:  |  Size: 149 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 137 KiB

After

Width:  |  Height:  |  Size: 137 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 138 KiB

After

Width:  |  Height:  |  Size: 161 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 138 KiB

After

Width:  |  Height:  |  Size: 161 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 183 KiB

After

Width:  |  Height:  |  Size: 193 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 184 KiB

After

Width:  |  Height:  |  Size: 194 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 78 KiB

After

Width:  |  Height:  |  Size: 82 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 107 KiB

After

Width:  |  Height:  |  Size: 117 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 117 KiB

After

Width:  |  Height:  |  Size: 125 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 119 KiB

After

Width:  |  Height:  |  Size: 126 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 196 KiB

After

Width:  |  Height:  |  Size: 202 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

After

Width:  |  Height:  |  Size: 46 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 106 KiB

After

Width:  |  Height:  |  Size: 111 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 54 KiB

After

Width:  |  Height:  |  Size: 55 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 60 KiB

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 52 KiB

After

Width:  |  Height:  |  Size: 53 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 120 KiB

After

Width:  |  Height:  |  Size: 126 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 96 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 593 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 162 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 122 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 158 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 122 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 88 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 87 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 72 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 111 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 112 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 158 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 110 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 85 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 111 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 108 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 50 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 117 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 115 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 58 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 149 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 195 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 186 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 212 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 57 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 80 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 88 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 175 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 63 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 80 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 65 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 81 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 84 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 82 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 186 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 46 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 151 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 152 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 120 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 126 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 65 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 53 KiB

Some files were not shown because too many files have changed in this diff Show More