diff --git a/Makefile b/Makefile index 2f0e5c1..3ea01a3 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test-pg: test-v6only: NMON_TEST_V6ONLY=1 $(GO) test -race -tags 'integration $(TAGS)' ./... -gate-m%: +gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6: gate-m%: ./scripts/gate-m$*.sh clean: diff --git a/cmd/nmon/migrate.go b/cmd/nmon/migrate.go new file mode 100644 index 0000000..4d68bd7 --- /dev/null +++ b/cmd/nmon/migrate.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/v10networks/nmon/internal/clock" + "github.com/v10networks/nmon/internal/config" + "github.com/v10networks/nmon/internal/store" +) + +// migrate applies pending inventory migrations (up, the default) or reverts the +// latest group (down). Credentials are not touched, so no key file is needed. +func migrate(ctx context.Context, args []string) error { + action := "up" + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + action, args = args[0], args[1:] + } + if action != "up" && action != "down" { + return fmt.Errorf("unknown action %q: want up or down", action) + } + var f config.Flags + fs := newFlagSet("migrate", "migrate [up|down] --data-dir DIR [flags]") + storeFlags(fs, &f) + if err := fs.Parse(args); err != nil { + return flagErr(err) + } + if f.DataDir == "" { + return fmt.Errorf("--data-dir is required") + } + if err := os.MkdirAll(f.DataDir, 0o755); err != nil { + return err + } + dsn := f.DBDSN + if dsn == "" { + if f.DBDriver != store.DriverSQLite { + return fmt.Errorf("--db-dsn is required for --db-driver %s", f.DBDriver) + } + dsn = config.InventoryDB(f.DataDir) + } + st, err := store.Open(ctx, f.DBDriver, dsn, nil, clock.Real()) + if err != nil { + return err + } + defer st.Close() + if action == "down" { + if err := st.Rollback(ctx); err != nil { + return err + } + fmt.Println("reverted the latest migration group") + return nil + } + if err := st.Migrate(ctx); err != nil { + return err + } + fmt.Println("migrations up to date") + return nil +} diff --git a/cmd/nmon/serve.go b/cmd/nmon/serve.go new file mode 100644 index 0000000..fccde61 --- /dev/null +++ b/cmd/nmon/serve.go @@ -0,0 +1,77 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log/slog" + "os" + + "github.com/v10networks/nmon/internal/app" + "github.com/v10networks/nmon/internal/config" +) + +func init() { + register("serve", "run the monitor: collect, store and serve the API", serve) + register("migrate", "apply (up) or revert (down) inventory migrations", migrate) +} + +// hidden flags are dev/test only and left out of the usage text. +var hidden = map[string]bool{"poll-interval": true, "snmp-port": true} + +// newFlagSet returns a FlagSet whose usage skips hidden flags. +func newFlagSet(name, usage string) *flag.FlagSet { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "usage: nmon %s\n\nflags:\n", usage) + fs.VisitAll(func(f *flag.Flag) { + if hidden[f.Name] { + return + } + def := "" + if f.DefValue != "" && f.DefValue != "false" && f.DefValue != "0" { + def = fmt.Sprintf(" (default %s)", f.DefValue) + } + fmt.Fprintf(fs.Output(), " --%s\n %s%s\n", f.Name, f.Usage, def) + }) + } + return fs +} + +// storeFlags registers the flags every inventory-using command shares. +func storeFlags(fs *flag.FlagSet, f *config.Flags) { + fs.StringVar(&f.DataDir, "data-dir", "", "data folder: inventory.db, vm-hot/, vm-long/, bin/, profiles/, config.yaml (required)") + fs.StringVar(&f.DBDriver, "db-driver", "sqlite", "inventory database: sqlite, postgres or mysql") + fs.StringVar(&f.DBDSN, "db-dsn", "", "database DSN; default /inventory.db for sqlite") +} + +func serve(ctx context.Context, args []string) error { + var f config.Flags + fs := newFlagSet("serve", "serve --data-dir DIR --insecure-http [flags]") + storeFlags(fs, &f) + fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile, "SNMP credential key file; created on first run; must be outside --data-dir") + fs.StringVar(&f.Listen, "listen", app.DefaultListen, "address for the API") + fs.BoolVar(&f.InsecureHTTP, "insecure-http", false, "serve plain HTTP with no authentication (required until TLS and auth arrive in M6)") + fs.StringVar(&f.VMHotURL, "vm-hot-url", "", "use this VictoriaMetrics instance as the hot store instead of the embedded one") + fs.StringVar(&f.VMLongURL, "vm-long-url", "", "use this VictoriaMetrics instance as the long store instead of the embedded one") + fs.IntVar(&f.MaxInflight, "max-inflight", 512, "most SNMP polls in flight at once") + fs.DurationVar(&f.PollInterval, "poll-interval", 0, "dev: SNMP poll interval (default 60s)") + fs.IntVar(&f.SNMPPort, "snmp-port", 0, "dev: UDP port of every device's agent (default 161)") + if err := fs.Parse(args); err != nil { + return flagErr(err) + } + if f.DataDir == "" { + return fmt.Errorf("--data-dir is required") + } + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))) + slog.Info("nmon starting", "data_dir", f.DataDir, "listen", f.Listen) + return app.Run(ctx, f) +} + +// flagErr turns flag.ErrHelp into a clean exit. +func flagErr(err error) error { + if err == flag.ErrHelp { + os.Exit(0) + } + return err +} diff --git a/docs/milestones/M1-skeleton-and-fixtures.md b/docs/milestones/M1-skeleton-and-fixtures.md index d5532e6..10ecf10 100644 --- a/docs/milestones/M1-skeleton-and-fixtures.md +++ b/docs/milestones/M1-skeleton-and-fixtures.md @@ -90,4 +90,22 @@ WP2, WP3 and WP4 run in parallel. WP5 runs after them. ## Gate log -_(filled in by the supervisor when the gate passes)_ +**Passed 2026-10-22**, run by the supervisor (Opus) on linux/amd64 (16 cores, 15 GB), Go 1.27.1, VictoriaMetrics v1.153.0. + +| Check | Result | +|---|---| +| `gofmt -l`, `go vet ./...` | clean | +| `go test -race ./...` | 13 packages ok | +| `go test -race -tags 'integration vmembed' ./...` | all ok, including real-VM supervisor tests (kill -9 → healthy in about 1.4 s, listeners on 127.0.0.1 only) and in-process `app.Run` | +| `./scripts/test-pg.sh` (Postgres 16) | store suite ok on SQLite and Postgres; MySQL 8 suite also run once by WP2 | +| `make gate-m1` | `GATE M1: PASS`: static binary; counters fixture on `[::1]`; 8 `ifHCInOctets` series with labels exactly `{__name__, device, iface}`, all strictly rising; 2 store listeners, both on 127.0.0.1; clean SIGTERM exit | + +**Decisions taken in M1** (beyond DEVIATIONS 1–5): +- `POST /api/v1/devices` walks the device before writing. An unreachable device returns 502 and nothing is stored, so no slug is burned. +- The 100k-series query cap is enforced on the result (422), not with a store flag, because rollups need more headroom. +- Supervisor: 3 restarts in 10 min are allowed; the 4th failure raises `store_failed` and stops the loop. The children run with `-search.latencyOffset=0s`. +- The init migration uses frozen struct copies. Credential ciphertext is bound to its row (AAD includes the row id). +- The dev-only flags `--poll-interval` and `--snmp-port` exist for gates and tests. + +**Carried into M2:** a per-device SNMP port (a `devices.snmp_port` column replacing the global `--snmp-port`, which the loadtest needs); varbind adaptation, backoff and degraded state; the 30 s poll for 32-bit fast links; periodic re-discovery; full identity matching (`TODO(M2)` in `internal/discovery`). +**Carried into M3:** `/docs` currently loads Swagger UI assets from a CDN; self-host them with the embedded UI. diff --git a/docs/milestones/README.md b/docs/milestones/README.md index e1cb97c..ea87e07 100644 --- a/docs/milestones/README.md +++ b/docs/milestones/README.md @@ -11,8 +11,8 @@ its **Gate log** is filled in when the gate passes. A milestone is done only whe | # | Milestone | Gate | Doc | Status | |---|---|---|---|---| -| M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | in progress | -| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | planned | +| M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | **passed**, tagged `m1` | +| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | in progress (WP1 done) | | M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | planned | | M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | planned | | M5 | Push ingest, rollup, checks | a replayed Proxmox sample shows under its node and a 2 y query hits the 1 h tier | [M5](M5-ingest-rollup-checks.md) | planned | diff --git a/go.mod b/go.mod index 6cc367c..e910205 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/v10networks/nmon go 1.27.1 require ( + github.com/danielgtaylor/huma/v2 v2.39.1 github.com/go-sql-driver/mysql v1.10.1 github.com/gosnmp/gosnmp v1.46.0 github.com/oklog/ulid/v2 v2.1.2 diff --git a/go.sum b/go.sum index f7cc524..3e17878 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/danielgtaylor/huma/v2 v2.39.1 h1:0kwF4ltQoYZ+IU55VPy+BcGekzgF44R64daTGde1H+g= +github.com/danielgtaylor/huma/v2 v2.39.1/go.mod h1:zcnQ38duIJ3VUHwFaBoZ6x8T+KN/mr33oyqxcj0HTug= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= +github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78= +github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= +github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -51,6 +57,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= diff --git a/internal/api/api.go b/internal/api/api.go new file mode 100644 index 0000000..d4f02d0 --- /dev/null +++ b/internal/api/api.go @@ -0,0 +1,154 @@ +// Package api is nmon's HTTP API: huma v2 on net/http, errors as RFC 9457 +// problem details, OpenAPI generated from the handlers at /openapi.json. +package api + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "strconv" + "strings" + "time" + + "github.com/danielgtaylor/huma/v2" + "github.com/danielgtaylor/huma/v2/adapters/humago" + + "github.com/v10networks/nmon/internal/clock" + "github.com/v10networks/nmon/internal/selfmon" + "github.com/v10networks/nmon/internal/store" +) + +// Limits. +const ( + // QueryTimeout bounds one /query call. + QueryTimeout = 30 * time.Second + // MaxSeries is the most series a /query call may return; more is a 422. + // It is enforced on the result rather than with a store flag because the + // rollup job needs the store's own limit far higher (-search.maxUniqueTimeseries). + MaxSeries = 100_000 + // DiscoveryTimeout bounds the seed discovery run by POST /devices. + DiscoveryTimeout = 30 * time.Second +) + +// Querier runs read-only MetricsQL against the hot store. *vmsup.Querier implements it. +type Querier interface { + Query(ctx context.Context, store, q string, ts time.Time) ([]byte, error) + QueryRange(ctx context.Context, store, q string, from, to time.Time, step time.Duration) ([]byte, error) +} + +// Refresher is told when the device list changed. *poller.Poller implements it. +type Refresher interface{ Refresh() } + +// Deps are what the handlers need. Store, Querier and Poller are required. +type Deps struct { + Store store.Store + Querier Querier + Poller Refresher + Clock clock.Clock // default clock.Real() + Registry *selfmon.Registry // served at /metrics; default selfmon.Default + Logger *slog.Logger // default slog.Default() + SNMPPort int // dev/test override of UDP 161; 0 = 161 + + // DiscoveryTimeout overrides the 30 s seed discovery limit (tests). + DiscoveryTimeout time.Duration +} + +type server struct { + Deps +} + +// New returns the API handler with every route registered. +func New(d Deps) http.Handler { + if d.Clock == nil { + d.Clock = clock.Real() + } + if d.Registry == nil { + d.Registry = selfmon.Default + } + if d.Logger == nil { + d.Logger = slog.Default() + } + if d.DiscoveryTimeout <= 0 { + d.DiscoveryTimeout = DiscoveryTimeout + } + s := &server{d} + + mux := http.NewServeMux() + cfg := huma.DefaultConfig("nmon", "1.0.0") + cfg.Info.Description = "nmon network monitor API." + cfg.OpenAPIPath = "/openapi" + cfg.DocsPath = "/docs" + cfg.DocsRenderer = huma.DocsRendererSwaggerUI + cfg.CreateHooks = nil // no $schema fields or Link headers in responses + api := humago.New(mux, cfg) + + s.registerDevices(api) + s.registerQuery(api) + s.registerMetrics(api) + return problemFallback(mux) +} + +// storeErr maps store sentinels to problem responses; anything else is a 500 +// whose detail does not leak internals. +func (s *server) storeErr(err error) error { + switch { + case errors.Is(err, store.ErrNotFound): + return huma.Error404NotFound("not found") + case errors.Is(err, store.ErrConflict): + return huma.Error409Conflict(trimSentinel(err, store.ErrConflict)) + case errors.Is(err, store.ErrInvalid): + return huma.Error422UnprocessableEntity(trimSentinel(err, store.ErrInvalid)) + } + s.Logger.Error("store error", "err", err) + return huma.Error500InternalServerError("internal error") +} + +// trimSentinel returns err's text without the leading "store: x: " sentinel. +func trimSentinel(err, sentinel error) string { + msg := err.Error() + if rest, ok := strings.CutPrefix(msg, sentinel.Error()+": "); ok { + return rest + } + return strings.TrimPrefix(msg, "store: ") +} + +// problemFallback answers requests that match no route (404, 405) with +// problem+json instead of the mux's plain text. +func problemFallback(mux *http.ServeMux) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h, pattern := mux.Handler(r) + if pattern != "" { + mux.ServeHTTP(w, r) // sets the path values + return + } + cw := &capture{h: http.Header{}} + h.ServeHTTP(cw, r) + status := cw.status + if status == 0 { + status = http.StatusNotFound + } + if allow := cw.h.Get("Allow"); allow != "" { + w.Header().Set("Allow", allow) + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{ + "title": http.StatusText(status), "status": status, + "detail": r.Method + " " + r.URL.Path + ": " + strings.ToLower(http.StatusText(status)), + }) + }) +} + +// capture records only the status and headers the mux would have written. +type capture struct { + h http.Header + status int +} + +func (c *capture) Header() http.Header { return c.h } +func (c *capture) Write(b []byte) (int, error) { return len(b), nil } +func (c *capture) WriteHeader(s int) { c.status = s } + +func itoa(n int) string { return strconv.Itoa(n) } diff --git a/internal/api/api_test.go b/internal/api/api_test.go new file mode 100644 index 0000000..5cb30cd --- /dev/null +++ b/internal/api/api_test.go @@ -0,0 +1,272 @@ +package api_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/v10networks/nmon/internal/api" + "github.com/v10networks/nmon/internal/clock" + "github.com/v10networks/nmon/internal/fixturegen" + "github.com/v10networks/nmon/internal/selfmon" + "github.com/v10networks/nmon/internal/simagent" + "github.com/v10networks/nmon/internal/store" + "github.com/v10networks/nmon/internal/testutil" + "github.com/v10networks/nmon/internal/ulid" + "github.com/v10networks/nmon/internal/vmsup" +) + +type querier struct { + body []byte + err error + q string + ts, from, to time.Time + step time.Duration + instant, rangeCnt int +} + +func (q *querier) Query(_ context.Context, st, expr string, ts time.Time) ([]byte, error) { + q.instant++ + q.q, q.ts = expr, ts + return q.body, q.err +} + +func (q *querier) QueryRange(_ context.Context, st, expr string, from, to time.Time, step time.Duration) ([]byte, error) { + q.rangeCnt++ + q.q, q.from, q.to, q.step = expr, from, to, step + return q.body, q.err +} + +type refresher struct{ n atomic.Int32 } + +func (r *refresher) Refresh() { r.n.Add(1) } + +type env struct { + h http.Handler + st store.Store + q *querier + ref *refresher + reg *selfmon.Registry +} + +func newEnv(t *testing.T, port int) *env { + clk := clock.NewFake(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)) + e := &env{st: testutil.OpenStore(t, clk), q: &querier{body: []byte(`{"status":"success","data":{"resultType":"vector","result":[]}}`)}, + ref: &refresher{}, reg: selfmon.New()} + e.h = api.New(api.Deps{Store: e.st, Querier: e.q, Poller: e.ref, Clock: clk, Registry: e.reg, SNMPPort: port, + DiscoveryTimeout: 3 * time.Second}) + return e +} + +func (e *env) do(method, path, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + rec := httptest.NewRecorder() + e.h.ServeHTTP(rec, req) + return rec +} + +func isProblem(t *testing.T, rec *httptest.ResponseRecorder, status int) { + t.Helper() + if rec.Code != status { + t.Fatalf("status %d, want %d: %s", rec.Code, status, rec.Body) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/problem+json") { + t.Fatalf("content type %q, want problem+json: %s", ct, rec.Body) + } + var p struct { + Title string `json:"title"` + Status int `json:"status"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &p); err != nil || p.Status != status || p.Title == "" { + t.Fatalf("problem body %s (%v)", rec.Body, err) + } +} + +const community = "s3cr3t-comm-xyz" + +func TestAddAndGetDevice(t *testing.T) { + a := testutil.StartAgent(t, simagent.Config{ + Recording: fixturegen.Generic(fixturegen.Options{Ports: 8, SysName: "edge1", HCCounters: true}), Community: community, + }) + e := newEnv(t, int(a.Addr().Port())) + body := fmt.Sprintf(`{"name":"Edge One","addresses":["::1"],"site":"yyz","role":"edge","snmp":{"version":"v2c","community":%q}}`, community) + rec := e.do("POST", "/api/v1/devices", body) + if rec.Code != http.StatusCreated { + t.Fatalf("POST: %d %s", rec.Code, rec.Body) + } + if e.ref.n.Load() != 1 { + t.Error("poller not refreshed") + } + var created api.DeviceView + if err := json.Unmarshal(rec.Body.Bytes(), &created); err != nil { + t.Fatal(err) + } + if created.Slug == "" || created.Name != "Edge One" || created.Site != "yyz" || created.SysName != "edge1" || + created.MgmtIP != "::1" || created.SNMPVersion != "v2c" || len(created.Interfaces) != 8 { + t.Fatalf("created: %+v", created) + } + + get := e.do("GET", "/api/v1/devices/"+created.Slug, "") + if get.Code != 200 { + t.Fatalf("GET: %d %s", get.Code, get.Body) + } + for _, r := range []*httptest.ResponseRecorder{rec, get} { + if strings.Contains(r.Body.String(), community) { + t.Fatalf("response contains the community: %s", r.Body) + } + var m map[string]json.RawMessage + if err := json.Unmarshal(r.Body.Bytes(), &m); err != nil { + t.Fatal(err) + } + for _, k := range []string{"id", "snmp_credential_id", "community", "device_id"} { + if _, ok := m[k]; ok { + t.Errorf("device response has key %q", k) + } + } + } + var got api.DeviceView + if err := json.Unmarshal(get.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + for _, i := range got.Interfaces { + if !ulid.Valid(i.ID) { + t.Errorf("interface id %q", i.ID) + } + if i.FirstSeen.Location() != time.UTC { + t.Errorf("timestamp not UTC: %v", i.FirstSeen) + } + } + if !strings.Contains(get.Body.String(), `"created_at":"2026-10-22T12:00:00Z"`) { + t.Errorf("created_at is not RFC 3339 UTC: %s", get.Body) + } + // the secret is in the store, encrypted, and decrypts for the poller + d, _ := e.st.GetDeviceBySlug(context.Background(), created.Slug) + c, err := e.st.GetCredential(context.Background(), *d.SNMPCredentialID) + if err != nil || c.Community != community { + t.Fatalf("stored credential: %+v %v", c, err) + } +} + +func TestAddDeviceRejections(t *testing.T) { + e := newEnv(t, 1) // nothing listens on UDP 1 + for name, tc := range map[string]struct { + body string + status int + }{ + "no snmp for network": {`{"name":"x","addresses":["::1"]}`, 422}, + "bad address": {`{"name":"x","addresses":["nope"],"snmp":{"version":"v2c","community":"c"}}`, 422}, + "no community": {`{"name":"x","addresses":["::1"],"snmp":{"version":"v2c"}}`, 422}, + "bad version": {`{"name":"x","addresses":["::1"],"snmp":{"version":"v9","community":"c"}}`, 422}, + "no addresses": {`{"name":"x","addresses":[],"snmp":{"version":"v2c","community":"c"}}`, 422}, + "unreachable": {`{"name":"x","addresses":["::1"],"snmp":{"version":"v2c","community":"c"}}`, 502}, + } { + t.Run(name, func(t *testing.T) { isProblem(t, e.do("POST", "/api/v1/devices", tc.body), tc.status) }) + } + if _, n, err := e.st.ListDevices(context.Background(), store.DeviceFilter{}, store.Page{}); err != nil || n != 0 { + t.Fatalf("rejected requests stored %d devices (%v)", n, err) + } + if e.ref.n.Load() != 0 { + t.Error("poller refreshed for a rejected device") + } + // a device without SNMP is fine + rec := e.do("POST", "/api/v1/devices", `{"name":"vm1","kind":"vm","addresses":["192.0.2.7"]}`) + if rec.Code != 201 { + t.Fatalf("vm: %d %s", rec.Code, rec.Body) + } +} + +func TestProblemDetails(t *testing.T) { + e := newEnv(t, 0) + isProblem(t, e.do("GET", "/api/v1/devices/no-such-device", ""), 404) + isProblem(t, e.do("GET", "/api/v1/nothing", ""), 404) + rec := e.do("DELETE", "/api/v1/query", "") + isProblem(t, rec, 405) + isProblem(t, e.do("GET", "/api/v1/query", ""), 422) // missing q +} + +func TestOpenAPIDocsAndMetrics(t *testing.T) { + e := newEnv(t, 0) + e.reg.Counter("nmon_test_total", "device", "d1").Add(3) + rec := e.do("GET", "/openapi.json", "") + var spec struct { + Paths map[string]map[string]any `json:"paths"` + } + if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &spec) != nil { + t.Fatalf("openapi.json: %d %s", rec.Code, rec.Body) + } + for _, p := range []string{"/api/v1/devices", "/api/v1/devices/{slug}", "/api/v1/query", "/api/v1/query_range", "/metrics"} { + if spec.Paths[p] == nil { + t.Errorf("openapi.json lacks %s", p) + } + } + if d := e.do("GET", "/docs", ""); d.Code != 200 || !strings.Contains(d.Header().Get("Content-Type"), "text/html") { + t.Errorf("/docs: %d %s", d.Code, d.Header().Get("Content-Type")) + } + m := e.do("GET", "/metrics", "") + if m.Code != 200 || !strings.Contains(m.Body.String(), `nmon_test_total{device="d1"} 3`) { + t.Errorf("/metrics: %d %s", m.Code, m.Body) + } +} + +func seriesBody(n int) []byte { + var b bytes.Buffer + b.WriteString(`{"status":"success","data":{"resultType":"vector","result":[`) + for i := 0; i < n; i++ { + if i > 0 { + b.WriteByte(',') + } + b.WriteString(`{"metric":{},"value":[1,"1"]}`) + } + b.WriteString(`]}}`) + return b.Bytes() +} + +func TestQueryProxy(t *testing.T) { + e := newEnv(t, 0) + e.q.body = seriesBody(3) + rec := e.do("GET", "/api/v1/query?q=ifHCInOctets&time=2026-10-22T12:00:00Z", "") + if rec.Code != 200 || !bytes.Equal(rec.Body.Bytes(), e.q.body) || rec.Header().Get("Content-Type") != "application/json" { + t.Fatalf("passthrough: %d %s", rec.Code, rec.Body) + } + if e.q.q != "ifHCInOctets" || !e.q.ts.Equal(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)) { + t.Errorf("forwarded %q at %v", e.q.q, e.q.ts) + } + e.do("GET", "/api/v1/query?q=up&time=1700000000.5", "") + if e.q.ts.UnixMilli() != 1700000000500 { + t.Errorf("unix time parsed as %v", e.q.ts) + } + isProblem(t, e.do("GET", "/api/v1/query?q=up&time=yesterday", ""), 422) + + rec = e.do("GET", "/api/v1/query_range?q=up&from=1700000000&to=1700003600&step=30s", "") + if rec.Code != 200 || e.q.step != 30*time.Second || e.q.to.Sub(e.q.from) != time.Hour { + t.Fatalf("range: %d %s step=%v", rec.Code, rec.Body, e.q.step) + } + isProblem(t, e.do("GET", "/api/v1/query_range?q=up", ""), 422) // step required + isProblem(t, e.do("GET", "/api/v1/query_range?q=up&step=0", ""), 422) + + e.q.body = seriesBody(api.MaxSeries) + if rec := e.do("GET", "/api/v1/query?q=x", ""); rec.Code != 200 { + t.Fatalf("exactly %d series must pass: %d", api.MaxSeries, rec.Code) + } + e.q.body = seriesBody(api.MaxSeries + 1) + isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 422) + isProblem(t, e.do("GET", "/api/v1/query_range?q=x&step=1m", ""), 422) + + e.q.err = fmt.Errorf("%w: hot", vmsup.ErrStoreDown) + isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 503) + e.q.err = errors.New("vmsup: hot store: 422 Unprocessable Entity: cannot parse") + isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 422) + e.q.err = errors.New("connection reset") + isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 502) +} diff --git a/internal/api/devices.go b/internal/api/devices.go new file mode 100644 index 0000000..227ed86 --- /dev/null +++ b/internal/api/devices.go @@ -0,0 +1,281 @@ +package api + +import ( + "context" + "fmt" + "net/netip" + "time" + + "github.com/danielgtaylor/huma/v2" + + "github.com/v10networks/nmon/internal/discovery" + "github.com/v10networks/nmon/internal/snmp" + "github.com/v10networks/nmon/internal/store" + "github.com/v10networks/nmon/internal/ulid" +) + +// SNMPBody is the SNMP credential in a request. Secrets are write-only: no +// response ever contains them. +type SNMPBody struct { + Version string `json:"version" enum:"v1,v2c,v3" doc:"SNMP version."` + Community string `json:"community,omitempty" doc:"v1 and v2c community." example:"public"` + V3User string `json:"v3_user,omitempty" doc:"v3 user name."` + V3AuthProto string `json:"v3_auth_proto,omitempty" doc:"MD5, SHA, SHA224, SHA256 (default), SHA384 or SHA512."` + V3AuthPass string `json:"v3_auth_pass,omitempty"` + V3PrivProto string `json:"v3_priv_proto,omitempty" doc:"DES, AES (default), AES192 or AES256."` + V3PrivPass string `json:"v3_priv_pass,omitempty"` + V3Context string `json:"v3_context,omitempty"` +} + +// AddDeviceBody is the body of POST /api/v1/devices. +type AddDeviceBody struct { + Name string `json:"name" minLength:"1" maxLength:"255" example:"core-sw1"` + Addresses []string `json:"addresses" minItems:"1" doc:"Management addresses; an IPv6 address is preferred over IPv4." example:"[\"2001:db8::1\"]"` + Kind string `json:"kind,omitempty" enum:"network,vm,container,check_only" default:"network"` + Site string `json:"site,omitempty"` + Role string `json:"role,omitempty"` + SNMP *SNMPBody `json:"snmp,omitempty" doc:"Required for kind network."` +} + +// InterfaceView is an interface as the API shows it, addressed by ULID. +type InterfaceView struct { + ID string `json:"id" doc:"Immutable ULID."` + IfIndex int64 `json:"ifindex" doc:"Current ifIndex; may change, the ID does not."` + IfName string `json:"ifname"` + IfDescr string `json:"ifdescr"` + IfAlias string `json:"ifalias"` + IfPhysAddr string `json:"ifphysaddr"` + IfSpeed int64 `json:"ifspeed" doc:"bits per second"` + IfHighSpeed int64 `json:"ifhighspeed" doc:"megabits per second"` + IfType int64 `json:"iftype"` + AdminStatus int64 `json:"admin_status"` + OperStatus int64 `json:"oper_status"` + HasHCCounters bool `json:"has_hc_counters"` + Excluded bool `json:"excluded"` + FirstSeen time.Time `json:"first_seen"` + LastDiscovered time.Time `json:"last_discovered"` +} + +// DeviceView is a device as the API shows it. It carries no numeric ID and no credential. +type DeviceView struct { + Slug string `json:"slug" doc:"Immutable public key."` + Name string `json:"name"` + Kind string `json:"kind"` + Site string `json:"site"` + Role string `json:"role"` + Profile string `json:"profile"` + Enabled bool `json:"enabled"` + MgmtIP string `json:"mgmt_ip"` + Addresses []string `json:"addresses"` + SNMPVersion string `json:"snmp_version,omitempty"` + SysName string `json:"sys_name"` + SysObjectID string `json:"sys_object_id"` + SysDescr string `json:"sys_descr"` + PollState string `json:"poll_state"` + Warnings []string `json:"warnings"` + LastSeen *time.Time `json:"last_seen,omitempty"` + CreatedAt time.Time `json:"created_at"` + RetiredAt *time.Time `json:"retired_at,omitempty"` + Interfaces []InterfaceView `json:"interfaces"` +} + +type deviceOut struct{ Body DeviceView } + +func (s *server) registerDevices(api huma.API) { + huma.Register(api, huma.Operation{ + OperationID: "add-device", Method: "POST", Path: "/api/v1/devices", Tags: []string{"devices"}, + Summary: "Add a device", + Description: "Creates the credential, device and addresses, runs seed discovery synchronously (30 s limit) and starts polling. A device that does not answer SNMP is rejected with 502 and nothing is stored.", + DefaultStatus: 201, + Errors: []int{409, 422, 502}, + }, s.addDevice) + huma.Register(api, huma.Operation{ + OperationID: "get-device", Method: "GET", Path: "/api/v1/devices/{slug}", Tags: []string{"devices"}, + Summary: "Get a device and its interfaces", + Errors: []int{404}, + }, s.getDevice) +} + +func (s *server) addDevice(ctx context.Context, in *struct{ Body AddDeviceBody }) (*deviceOut, error) { + b := in.Body + kind := b.Kind + if kind == "" { + kind = store.KindNetwork + } + var addrs []netip.Addr + for _, a := range b.Addresses { + ip, err := netip.ParseAddr(a) + if err != nil { + return nil, huma.Error422UnprocessableEntity(fmt.Sprintf("address %q is not an IP address", a)) + } + addrs = append(addrs, ip.WithZone("").Unmap()) + } + if len(addrs) == 0 { + return nil, huma.Error422UnprocessableEntity("at least one address is required") + } + if kind == store.KindNetwork && b.SNMP == nil { + return nil, huma.Error422UnprocessableEntity("snmp credentials are required for kind network") + } + var ( + cred *store.SNMPCredential + res *discovery.Result + ) + if b.SNMP != nil { + var err error + if cred, err = s.credential(b.SNMP); err != nil { + return nil, err + } + pref := addrs[0] + for _, a := range addrs { + if a.Is6() { + pref = a + break + } + } + if res, err = s.collect(ctx, pref, cred); err != nil { + return nil, err + } + } + + var slug string + err := s.Store.WithTx(ctx, func(tx store.Store) error { + d := &store.Device{Name: b.Name, Kind: kind, Site: b.Site, Role: b.Role, Enabled: true, Profile: "generic"} + if cred != nil { + if err := tx.CreateCredential(ctx, cred); err != nil { + return err + } + d.SNMPCredentialID = &cred.ID + } + if err := tx.CreateDevice(ctx, d); err != nil { + return err + } + strs := make([]string, len(addrs)) + for i, a := range addrs { + strs[i] = a.String() + } + if err := tx.SetAddresses(ctx, d.ID, strs); err != nil { + return err + } + if res != nil { + if err := discovery.Apply(ctx, tx, d, res, s.Clock); err != nil { + return err + } + } + slug = d.Slug + return tx.AppendAudit(ctx, "anonymous", "device.add", d.Slug, nil, + map[string]any{"name": d.Name, "kind": d.Kind, "addresses": strs}) + }) + if err != nil { + return nil, s.storeErr(err) + } + s.Poller.Refresh() + return s.view(ctx, slug) +} + +// credential validates the request's SNMP block and returns a fresh credential. +func (s *server) credential(b *SNMPBody) (*store.SNMPCredential, error) { + c := &store.SNMPCredential{ + Name: "cred-" + ulid.New(s.Clock.Now()), Version: b.Version, Community: b.Community, V3User: b.V3User, + V3AuthProto: b.V3AuthProto, V3AuthPass: b.V3AuthPass, V3PrivProto: b.V3PrivProto, V3PrivPass: b.V3PrivPass, + V3Context: b.V3Context, + } + switch b.Version { + case "v1", "v2c": + if b.Community == "" { + return nil, huma.Error422UnprocessableEntity("snmp.community is required for " + b.Version) + } + case "v3": + if b.V3User == "" { + return nil, huma.Error422UnprocessableEntity("snmp.v3_user is required for v3") + } + if _, err := snmp.ParseAuthProto(b.V3AuthProto); err != nil { + return nil, huma.Error422UnprocessableEntity(err.Error()) + } + if _, err := snmp.ParsePrivProto(b.V3PrivProto); err != nil { + return nil, huma.Error422UnprocessableEntity(err.Error()) + } + default: + return nil, huma.Error422UnprocessableEntity("snmp.version must be v1, v2c or v3") + } + return c, nil +} + +// collect runs the seed walk against addr before anything is stored. +func (s *server) collect(ctx context.Context, addr netip.Addr, cred *store.SNMPCredential) (*discovery.Result, error) { + dctx, cancel := context.WithTimeout(ctx, s.DiscoveryTimeout) + defer cancel() + sess, err := snmp.Dial(dctx, discovery.Target(addr, s.SNMPPort, cred, s.Clock), "discovery") + if err != nil { + return nil, huma.Error422UnprocessableEntity("snmp: " + err.Error()) + } + defer sess.Close() + res, err := discovery.Collect(dctx, sess, s.Clock) + if err != nil { + return nil, huma.Error502BadGateway("discovery of " + addr.String() + " failed: " + err.Error()) + } + return res, nil +} + +func (s *server) getDevice(ctx context.Context, in *struct { + Slug string `path:"slug" doc:"Device slug."` +}) (*deviceOut, error) { + return s.view(ctx, in.Slug) +} + +func (s *server) view(ctx context.Context, slug string) (*deviceOut, error) { + d, err := s.Store.GetDeviceBySlug(ctx, slug) + if err != nil { + return nil, s.storeErr(err) + } + if d.PurgedAt != nil { + return nil, huma.Error404NotFound("not found") + } + v := DeviceView{ + Slug: d.Slug, Name: d.Name, Kind: d.Kind, Site: d.Site, Role: d.Role, Profile: d.Profile, Enabled: d.Enabled, + MgmtIP: d.MgmtIP, SysName: d.SysName, SysObjectID: d.SysObjectID, SysDescr: d.SysDescr, + PollState: d.PollState, Warnings: nonNil(d.Warnings), LastSeen: utcp(d.LastSeen), CreatedAt: d.CreatedAt.UTC(), + RetiredAt: utcp(d.RetiredAt), Addresses: []string{}, Interfaces: []InterfaceView{}, + } + addrs, err := s.Store.ListAddresses(ctx, d.ID) + if err != nil { + return nil, s.storeErr(err) + } + for _, a := range addrs { + v.Addresses = append(v.Addresses, a.Addr) + } + if d.SNMPCredentialID != nil { + c, err := s.Store.GetCredential(ctx, *d.SNMPCredentialID) + if err != nil { + return nil, s.storeErr(err) + } + v.SNMPVersion = c.Version // the secrets stay behind + } + ifs, err := s.Store.ListInterfaces(ctx, d.ID, false) + if err != nil { + return nil, s.storeErr(err) + } + for _, i := range ifs { + v.Interfaces = append(v.Interfaces, InterfaceView{ + ID: i.ID, IfIndex: i.IfIndex, IfName: i.IfName, IfDescr: i.IfDescr, IfAlias: i.IfAlias, + IfPhysAddr: i.IfPhysAddr, IfSpeed: i.IfSpeed, IfHighSpeed: i.IfHighSpeed, IfType: i.IfType, + AdminStatus: i.AdminStatus, OperStatus: i.OperStatus, HasHCCounters: i.HasHCCounters, Excluded: i.Excluded, + FirstSeen: i.FirstSeen.UTC(), LastDiscovered: i.LastDiscovered.UTC(), + }) + } + return &deviceOut{Body: v}, nil +} + +func nonNil(s []string) []string { + if s == nil { + return []string{} + } + return s +} + +func utcp(t *time.Time) *time.Time { + if t == nil { + return nil + } + u := t.UTC() + return &u +} diff --git a/internal/api/errors_test.go b/internal/api/errors_test.go new file mode 100644 index 0000000..733ea23 --- /dev/null +++ b/internal/api/errors_test.go @@ -0,0 +1,29 @@ +package api + +import ( + "fmt" + "testing" + + "github.com/danielgtaylor/huma/v2" + + "github.com/v10networks/nmon/internal/store" +) + +func TestStoreErrorMapping(t *testing.T) { + s := &server{} + s.Logger = nil + for _, tc := range []struct { + err error + status int + detail string + }{ + {fmt.Errorf("%w: slug taken", store.ErrConflict), 409, "slug taken"}, + {fmt.Errorf("%w: bad kind", store.ErrInvalid), 422, "bad kind"}, + {fmt.Errorf("wrapped: %w", store.ErrNotFound), 404, "not found"}, + } { + se, ok := s.storeErr(tc.err).(huma.StatusError) + if !ok || se.GetStatus() != tc.status || se.Error() != tc.detail { + t.Errorf("%v: got %v, want %d %q", tc.err, se, tc.status, tc.detail) + } + } +} diff --git a/internal/api/query.go b/internal/api/query.go new file mode 100644 index 0000000..ea86bfa --- /dev/null +++ b/internal/api/query.go @@ -0,0 +1,169 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/danielgtaylor/huma/v2" + + "github.com/v10networks/nmon/internal/vmsup" +) + +var jsonObject = map[string]*huma.Response{ + "200": {Description: "The store's Prometheus-compatible JSON response, unchanged.", Content: map[string]*huma.MediaType{ + "application/json": {Schema: &huma.Schema{Type: "object"}}, + }}, +} + +func (s *server) registerQuery(api huma.API) { + huma.Register(api, huma.Operation{ + OperationID: "query", Method: "GET", Path: "/api/v1/query", Tags: []string{"query"}, + Summary: "Instant MetricsQL query against the hot store", + Description: "Read-only proxy with a 30 s timeout. A result of more than 100,000 series is rejected with 422.", + Responses: jsonObject, Errors: []int{422, 503, 504}, + }, s.query) + huma.Register(api, huma.Operation{ + OperationID: "query-range", Method: "GET", Path: "/api/v1/query_range", Tags: []string{"query"}, + Summary: "Range MetricsQL query against the hot store", + Description: "Same limits as /api/v1/query.", + Responses: jsonObject, Errors: []int{422, 503, 504}, + }, s.queryRange) +} + +func raw(body []byte) *huma.StreamResponse { + return &huma.StreamResponse{Body: func(ctx huma.Context) { + ctx.SetHeader("Content-Type", "application/json") + ctx.SetStatus(http.StatusOK) + _, _ = ctx.BodyWriter().Write(body) + }} +} + +func (s *server) query(ctx context.Context, in *struct { + Q string `query:"q" required:"true" minLength:"1" doc:"MetricsQL expression." example:"ifHCInOctets"` + Time string `query:"time" doc:"Evaluation time: RFC 3339 or Unix seconds. Default now."` +}) (*huma.StreamResponse, error) { + var ts time.Time + if in.Time != "" { + var err error + if ts, err = parseTime(in.Time); err != nil { + return nil, huma.Error422UnprocessableEntity("time: " + err.Error()) + } + } + qctx, cancel := context.WithTimeout(ctx, QueryTimeout) + defer cancel() + body, err := s.Querier.Query(qctx, "hot", in.Q, ts) + return s.finish(qctx, body, err) +} + +func (s *server) queryRange(ctx context.Context, in *struct { + Q string `query:"q" required:"true" minLength:"1" doc:"MetricsQL expression."` + From string `query:"from" doc:"Start: RFC 3339 or Unix seconds. Default one hour before to."` + To string `query:"to" doc:"End: RFC 3339 or Unix seconds. Default now."` + Step string `query:"step" required:"true" doc:"Resolution step: a duration such as 30s or a number of seconds."` +}) (*huma.StreamResponse, error) { + to := s.Clock.Now() + if in.To != "" { + var err error + if to, err = parseTime(in.To); err != nil { + return nil, huma.Error422UnprocessableEntity("to: " + err.Error()) + } + } + from := to.Add(-time.Hour) + if in.From != "" { + var err error + if from, err = parseTime(in.From); err != nil { + return nil, huma.Error422UnprocessableEntity("from: " + err.Error()) + } + } + step, err := parseStep(in.Step) + if err != nil { + return nil, huma.Error422UnprocessableEntity("step: " + err.Error()) + } + if !from.Before(to) { + return nil, huma.Error422UnprocessableEntity("from must be before to") + } + qctx, cancel := context.WithTimeout(ctx, QueryTimeout) + defer cancel() + body, err := s.Querier.QueryRange(qctx, "hot", in.Q, from, to, step) + return s.finish(qctx, body, err) +} + +// finish applies the series cap and maps store errors to problems. +func (s *server) finish(ctx context.Context, body []byte, err error) (*huma.StreamResponse, error) { + switch { + case err == nil: + case errors.Is(err, vmsup.ErrStoreDown): + return nil, huma.Error503ServiceUnavailable("the metric store is not available") + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + return nil, huma.Error504GatewayTimeout("query exceeded " + QueryTimeout.String()) + case strings.Contains(err.Error(), "400 Bad Request") || strings.Contains(err.Error(), "422 Unprocessable"): + return nil, huma.Error422UnprocessableEntity(err.Error()) // the store rejected the query + default: + s.Logger.Warn("query failed", "err", err) + return nil, huma.Error502BadGateway("metric store error") + } + if n := countSeries(body); n > MaxSeries { + return nil, huma.Error422UnprocessableEntity("query returned " + itoa(n) + " series; the limit is " + itoa(MaxSeries) + ", narrow the selector") + } + return raw(body), nil +} + +// countSeries returns len(data.result) of a Prometheus-style response. +func countSeries(body []byte) int { + var r struct { + Data struct { + Result []json.RawMessage `json:"result"` + } `json:"data"` + } + if json.NewDecoder(bytes.NewReader(body)).Decode(&r) != nil { + return 0 + } + return len(r.Data.Result) +} + +func parseTime(s string) (time.Time, error) { + if t, err := time.Parse(time.RFC3339, s); err == nil { + return t.UTC(), nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return time.Time{}, errors.New("want RFC 3339 or Unix seconds") + } + return time.UnixMilli(int64(f * 1000)).UTC(), nil +} + +func parseStep(s string) (time.Duration, error) { + d, err := time.ParseDuration(s) + if err != nil { + f, ferr := strconv.ParseFloat(s, 64) + if ferr != nil { + return 0, errors.New("want a duration such as 30s or a number of seconds") + } + d = time.Duration(f * float64(time.Second)) + } + if d <= 0 { + return 0, errors.New("must be positive") + } + return d, nil +} + +func (s *server) registerMetrics(api huma.API) { + huma.Register(api, huma.Operation{ + OperationID: "metrics", Method: "GET", Path: "/metrics", Tags: []string{"internals"}, + Summary: "nmon's own metrics in Prometheus text format", + Responses: map[string]*huma.Response{"200": {Description: "Prometheus text exposition.", Content: map[string]*huma.MediaType{ + "text/plain": {Schema: &huma.Schema{Type: "string"}}, + }}}, + }, func(context.Context, *struct{}) (*huma.StreamResponse, error) { + return &huma.StreamResponse{Body: func(ctx huma.Context) { + ctx.SetHeader("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + _ = s.Registry.WritePrometheus(ctx.BodyWriter()) + }}, nil + }) +} diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..aa59eee --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,263 @@ +// Package app wires nmon serve: config, secret key, inventory, metric stores, +// write queues, poller and HTTP API, and shuts them down in order. +package app + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/v10networks/nmon/internal/api" + "github.com/v10networks/nmon/internal/clock" + "github.com/v10networks/nmon/internal/config" + "github.com/v10networks/nmon/internal/poller" + "github.com/v10networks/nmon/internal/secrets" + "github.com/v10networks/nmon/internal/store" + "github.com/v10networks/nmon/internal/vmsup" +) + +// Defaults for flags left empty. +const ( + DefaultListen = "[::]:8443" + DefaultSecretKeyFile = "/etc/nmon/secret.key" +) + +const ( + startTimeout = 60 * time.Second + shutdownTimeout = 30 * time.Second +) + +// App is a running nmon server. +type App struct { + log *slog.Logger + st store.Store + sup *vmsup.Supervisor + hot *vmsup.Queue + long *vmsup.Queue + poller *poller.Poller + srv *http.Server + ln net.Listener + + bg context.CancelFunc // poller and queue goroutines + pollDone chan struct{} + queuesWG chan struct{} + serveErr chan error +} + +// Run starts nmon and blocks until ctx is cancelled or the HTTP server fails, +// then shuts everything down gracefully. +func Run(ctx context.Context, f config.Flags) error { + a, err := Start(ctx, f) + if err != nil { + return err + } + var runErr error + select { + case <-ctx.Done(): + case runErr = <-a.serveErr: + } + sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + return errors.Join(runErr, a.Stop(sctx)) +} + +// Start brings every component up and returns once the API is listening. ctx +// bounds startup only; call Stop to shut down. +func Start(ctx context.Context, f config.Flags) (_ *App, err error) { + log := slog.Default() + if !f.InsecureHTTP { + return nil, errors.New("TLS and authentication arrive in M6; until then pass --insecure-http to run without them (development only)") + } + if f.DataDir == "" { + return nil, errors.New("--data-dir is required") + } + if f.Listen == "" { + f.Listen = DefaultListen + } + if f.SecretKeyFile == "" { + f.SecretKeyFile = DefaultSecretKeyFile + } + if f.MaxInflight <= 0 { + f.MaxInflight = poller.DefaultMaxInflight + } + dataDir, err := filepath.Abs(f.DataDir) + if err != nil { + return nil, err + } + f.DataDir = dataDir + keyFile, err := filepath.Abs(f.SecretKeyFile) + if err != nil { + return nil, err + } + if inside(dataDir, keyFile) { + return nil, fmt.Errorf("--secret-key-file %s is inside --data-dir %s; a backup of the data dir must not contain the key", keyFile, dataDir) + } + if err := os.MkdirAll(dataDir, 0o755); err != nil { + return nil, err + } + cfg, err := config.Load(dataDir) + if err != nil { + return nil, err + } + + driver := f.DBDriver + if driver == "" { + driver = store.DriverSQLite + } + dsn := f.DBDSN + if dsn == "" { + if driver != store.DriverSQLite { + return nil, fmt.Errorf("--db-dsn is required for --db-driver %s", driver) + } + dsn = config.InventoryDB(dataDir) + } + if err := ensureKey(log, keyFile, driver == store.DriverSQLite && fileExists(config.InventoryDB(dataDir))); err != nil { + return nil, err + } + kr, err := secrets.LoadKeyring(keyFile) + if err != nil { + return nil, fmt.Errorf("secret key: %w", err) + } + + a := &App{log: log, serveErr: make(chan error, 1), pollDone: make(chan struct{}), queuesWG: make(chan struct{})} + defer func() { + if err != nil { + sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + _ = a.Stop(sctx) + } + }() + + if a.st, err = store.Open(ctx, driver, dsn, kr, clock.Real()); err != nil { + return nil, fmt.Errorf("open inventory: %w", err) + } + if err = a.st.Migrate(ctx); err != nil { + return nil, fmt.Errorf("migrate inventory: %w", err) + } + + a.sup, err = vmsup.New(vmsup.Options{ + DataDir: dataDir, HotURL: f.VMHotURL, LongURL: f.VMLongURL, + HotRetention: cfg.Retention.Hot.D(), LongRetention: cfg.Retention.Long.D(), Logger: log, + OnEvent: func(e vmsup.Event) { + log.Warn("metric store event", "type", e.Type, "store", e.Store, "detail", e.Detail, "restarts", e.Restarts) + }, + }) + if err != nil { + return nil, err + } + sctx, cancel := context.WithTimeout(ctx, startTimeout) + defer cancel() + if err = a.sup.Start(sctx); err != nil { + a.sup = nil // Start leaves nothing running when it fails + return nil, fmt.Errorf("start metric stores: %w", err) + } + log.Info("metric stores up", "hot", a.sup.HotURL(), "long", a.sup.LongURL()) + + bg, cancelBG := context.WithCancel(context.Background()) + a.bg = cancelBG + a.hot = vmsup.NewQueue(vmsup.StoreHot, a.sup, vmsup.QueueOptions{}) + a.long = vmsup.NewQueue(vmsup.StoreLong, a.sup, vmsup.QueueOptions{}) + qdone := make(chan struct{}, 2) + for _, q := range []*vmsup.Queue{a.hot, a.long} { + go func() { q.Run(bg); qdone <- struct{}{} }() + } + go func() { <-qdone; <-qdone; close(a.queuesWG) }() + + a.poller = poller.New(poller.Options{ + Store: a.st, Writer: a.hot, Interval: f.PollInterval, MaxInflight: f.MaxInflight, SNMPPort: f.SNMPPort, Logger: log, + }) + go func() { _ = a.poller.Run(bg); close(a.pollDone) }() + + handler := api.New(api.Deps{ + Store: a.st, Querier: vmsup.NewQuerier(a.sup), Poller: a.poller, SNMPPort: f.SNMPPort, Logger: log, + }) + if a.ln, err = net.Listen("tcp", f.Listen); err != nil { + return nil, fmt.Errorf("listen: %w", err) + } + a.srv = &http.Server{Handler: handler, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute} + go func() { + if err := a.srv.Serve(a.ln); !errors.Is(err, http.ErrServerClosed) { + a.serveErr <- err + } + }() + log.Warn("serving plain HTTP without authentication (--insecure-http); do not expose this port", "addr", a.ln.Addr().String()) + return a, nil +} + +// Addr is the address the API listens on. +func (a *App) Addr() string { return a.ln.Addr().String() } + +// HotURL is the base URL of the hot metric store. +func (a *App) HotURL() string { return a.sup.HotURL() } + +// Stop shuts down: HTTP first, then the poller, then the metric stores (which +// flushes the write queues), then the inventory. It is safe after a failed Start. +func (a *App) Stop(ctx context.Context) error { + var errs []error + if a.srv != nil { + errs = append(errs, a.srv.Shutdown(ctx)) + } + if a.bg != nil { + a.bg() + <-a.pollDone + <-a.queuesWG + } + if a.sup != nil { + errs = append(errs, a.sup.Stop(ctx)) // flushes both queues + } + if a.st != nil { + errs = append(errs, a.st.Close()) + } + a.log.Info("stopped") + return errors.Join(errs...) +} + +// ensureKey creates the key file on a first run. It refuses when an inventory +// already exists, since a new key could not decrypt its credentials. +func ensureKey(log *slog.Logger, path string, inventoryExists bool) error { + if _, err := os.Stat(path); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if inventoryExists { + return fmt.Errorf("secret key file %s is missing but the inventory exists; restore the key file (a new key cannot decrypt the stored SNMP credentials)", path) + } + if err := secrets.GenerateKeyFile(path); err != nil { + return fmt.Errorf("generate secret key: %w", err) + } + log.Error("GENERATED A NEW SECRET KEY: back it up now; without it the SNMP credentials in the inventory cannot be recovered", "path", path) + return nil +} + +// inside reports whether path is dir or below it. +func inside(dir, path string) bool { + rel, err := filepath.Rel(resolve(dir), resolve(path)) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// resolve follows symlinks in the deepest existing parent of p. +func resolve(p string) string { + rest := "" + for q := p; ; q = filepath.Dir(q) { + if r, err := filepath.EvalSymlinks(q); err == nil { + return filepath.Join(r, rest) + } + if filepath.Dir(q) == q { + return p + } + rest = filepath.Join(filepath.Base(q), rest) + } +} + +func fileExists(p string) bool { + _, err := os.Stat(p) + return err == nil +} diff --git a/internal/app/app_test.go b/internal/app/app_test.go new file mode 100644 index 0000000..3c4f357 --- /dev/null +++ b/internal/app/app_test.go @@ -0,0 +1,94 @@ +package app + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/v10networks/nmon/internal/config" + "github.com/v10networks/nmon/internal/secrets" +) + +func TestStartRefusals(t *testing.T) { + ctx := context.Background() + data := filepath.Join(t.TempDir(), "data") + outside := filepath.Join(t.TempDir(), "secret.key") + for name, tc := range map[string]struct { + f config.Flags + want string + }{ + "no insecure flag": {config.Flags{DataDir: data, SecretKeyFile: outside}, "--insecure-http"}, + "no data dir": {config.Flags{InsecureHTTP: true, SecretKeyFile: outside}, "--data-dir"}, + "key inside data": {config.Flags{DataDir: data, SecretKeyFile: filepath.Join(data, "keys", "secret.key"), InsecureHTTP: true}, "inside --data-dir"}, + "key is data dir": {config.Flags{DataDir: data, SecretKeyFile: data, InsecureHTTP: true}, "inside --data-dir"}, + } { + t.Run(name, func(t *testing.T) { + a, err := Start(ctx, tc.f) + if err == nil { + a.Stop(ctx) + t.Fatal("started") + } + if !strings.Contains(err.Error(), tc.want) { + t.Fatalf("error %q, want it to mention %q", err, tc.want) + } + }) + } + if _, err := os.Stat(outside); err == nil { + t.Error("a refused start created the key file") + } +} + +func TestStartRefusesMissingKeyWithExistingInventory(t *testing.T) { + data := t.TempDir() + if err := os.WriteFile(config.InventoryDB(data), nil, 0o600); err != nil { + t.Fatal(err) + } + key := filepath.Join(t.TempDir(), "secret.key") + _, err := Start(context.Background(), config.Flags{DataDir: data, SecretKeyFile: key, InsecureHTTP: true}) + if err == nil || !strings.Contains(err.Error(), "inventory exists") { + t.Fatalf("error %v", err) + } + if _, err := os.Stat(key); err == nil { + t.Error("a new key was generated next to an existing inventory") + } +} + +func TestEnsureKeyGeneratesOnce(t *testing.T) { + key := filepath.Join(t.TempDir(), "sub", "secret.key") + if err := ensureKey(testLogger(), key, false); err != nil { + t.Fatal(err) + } + st, err := os.Stat(key) + if err != nil || st.Mode().Perm() != 0o600 { + t.Fatalf("key file: %v %v", st, err) + } + if _, err := secrets.LoadKeyring(key); err != nil { + t.Fatal(err) + } + before, _ := os.ReadFile(key) + if err := ensureKey(testLogger(), key, true); err != nil { + t.Fatal(err) + } + if after, _ := os.ReadFile(key); string(after) != string(before) { + t.Error("existing key was replaced") + } +} + +func TestInside(t *testing.T) { + for _, tc := range []struct { + dir, path string + want bool + }{ + {"/var/lib/nmon", "/var/lib/nmon/secret.key", true}, + {"/var/lib/nmon", "/var/lib/nmon", true}, + {"/var/lib/nmon", "/var/lib/nmon2/secret.key", false}, + {"/var/lib/nmon", "/etc/nmon/secret.key", false}, + {"/var/lib/nmon", "/var/lib/secret.key", false}, + } { + if got := inside(tc.dir, tc.path); got != tc.want { + t.Errorf("inside(%s, %s) = %v", tc.dir, tc.path, got) + } + } +} diff --git a/internal/app/helpers_test.go b/internal/app/helpers_test.go new file mode 100644 index 0000000..c9aa0e5 --- /dev/null +++ b/internal/app/helpers_test.go @@ -0,0 +1,8 @@ +package app + +import ( + "io" + "log/slog" +) + +func testLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } diff --git a/internal/app/integration_test.go b/internal/app/integration_test.go new file mode 100644 index 0000000..952537e --- /dev/null +++ b/internal/app/integration_test.go @@ -0,0 +1,175 @@ +//go:build integration + +package app_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/v10networks/nmon/internal/app" + "github.com/v10networks/nmon/internal/config" + "github.com/v10networks/nmon/internal/simagent" + "github.com/v10networks/nmon/internal/testutil" + "github.com/v10networks/nmon/internal/vmsup" +) + +type series struct { + Metric map[string]string `json:"metric"` + Value [2]any `json:"value"` +} + +func query(t *testing.T, base, q string) []series { + t.Helper() + resp, err := http.Get(base + "/api/v1/query?q=" + q) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 { + t.Fatalf("query %s: %d %s", q, resp.StatusCode, body) + } + var r struct { + Data struct{ Result []series } `json:"data"` + } + if err := json.Unmarshal(body, &r); err != nil { + t.Fatal(err) + } + return r.Data.Result +} + +func TestServeCollectsCountersEndToEnd(t *testing.T) { + rec, err := simagent.LoadRecording("../../testdata/snmprec/counters.snmprec") + if err != nil { + t.Fatal(err) + } + agent := testutil.StartAgent(t, simagent.Config{Recording: rec, Community: "public"}) + + root := t.TempDir() + data := filepath.Join(root, "data") + a, err := app.Start(context.Background(), config.Flags{ + DataDir: data, SecretKeyFile: filepath.Join(root, "keys", "secret.key"), Listen: "[::1]:0", InsecureHTTP: true, + PollInterval: 5 * time.Second, SNMPPort: int(agent.Addr().Port()), + }) + if errors.Is(err, vmsup.ErrNoBinary) { + t.Skip("built without the vmembed tag") + } + if err != nil { + t.Fatal(err) + } + stopped := false + stop := func() { + if stopped { + return + } + stopped = true + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) + defer cancel() + if err := a.Stop(ctx); err != nil { + t.Errorf("stop: %v", err) + } + } + t.Cleanup(stop) + base := "http://" + a.Addr() + + resp, err := http.Post(base+"/api/v1/devices", "application/json", strings.NewReader( + `{"name":"counters","addresses":["::1"],"snmp":{"version":"v2c","community":"public"}}`)) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 201 { + t.Fatalf("add device: %d %s", resp.StatusCode, body) + } + var dev struct { + Slug string + Interfaces []struct{ ID string } + } + if err := json.Unmarshal(body, &dev); err != nil || len(dev.Interfaces) != 8 { + t.Fatalf("device: %s (%v)", body, err) + } + + // Instant queries return the newest point, so collect values of one counter + // across polls until three distinct ones have been seen. + var vals []float64 + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) && len(distinct(vals)) < 3 { + time.Sleep(2 * time.Second) + if r, err := http.Post(a.HotURL()+"/internal/force_flush", "", nil); err == nil { + r.Body.Close() + } + for _, s := range query(t, base, "ifHCInOctets") { + if s.Metric["iface"] == dev.Interfaces[0].ID { + v, _ := strconv.ParseFloat(s.Value[1].(string), 64) + vals = append(vals, v) + } + } + } + d := distinct(vals) + if len(d) < 3 { + t.Fatalf("saw only %v for ifHCInOctets after 60s", d) + } + for i := 1; i < len(d); i++ { + if d[i] <= d[i-1] { + t.Fatalf("not rising: %v", d) + } + } + for _, s := range query(t, base, "ifHCInOctets") { + keys := make([]string, 0, len(s.Metric)) + for k := range s.Metric { + keys = append(keys, k) + } + sort.Strings(keys) + if fmt.Sprint(keys) != "[__name__ device iface]" || s.Metric["device"] != dev.Slug { + t.Fatalf("labels %v", s.Metric) + } + } + if up := query(t, base, "nmon_snmp_up"); len(up) != 1 || up[0].Value[1] != "1" { + t.Errorf("nmon_snmp_up = %+v", up) + } + // the poller stores nmon_ internals every 15 s + stored := false + for end := time.Now().Add(30 * time.Second); time.Now().Before(end) && !stored; time.Sleep(2 * time.Second) { + if r, err := http.Post(a.HotURL()+"/internal/force_flush", "", nil); err == nil { + r.Body.Close() + } + stored = len(query(t, base, "nmon_pdus_total")) > 0 + } + if !stored { + t.Error("nmon_ internals are not stored") + } + if r, err := http.Get(base + "/metrics"); err != nil || r.StatusCode != 200 { + t.Errorf("/metrics: %v %v", r, err) + } else { + b, _ := io.ReadAll(r.Body) + r.Body.Close() + if !bytes.Contains(b, []byte("nmon_pdus_total")) { + t.Error("/metrics lacks nmon_pdus_total") + } + } + stop() +} + +func distinct(v []float64) []float64 { + s := append([]float64(nil), v...) + sort.Float64s(s) + out := s[:0] + for i, x := range s { + if i == 0 || x != s[i-1] { + out = append(out, x) + } + } + return out +} diff --git a/internal/config/config.go b/internal/config/config.go index e7d0c94..1895136 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,6 +25,7 @@ type Flags struct { VMLongURL string MaxInflight int PollInterval time.Duration // dev/test override of the 60 s interface interval; 0 = profile default + SNMPPort int // dev/test override of UDP 161 for every device; 0 = 161 } // Config is the content of config.yaml. diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go new file mode 100644 index 0000000..6ad6f38 --- /dev/null +++ b/internal/discovery/discovery.go @@ -0,0 +1,255 @@ +// Package discovery learns what a device has. In M1 it is the seed walk run +// when a device is added: system group plus the IF-MIB tables, creating +// interface rows with ULIDs. +// +// TODO(M2): identity matching beyond ifName and ifDescr+MAC (stack-member +// swaps, RMA), identity_reviews for ambiguous matches, retiring interfaces that +// disappeared, profiles, per-table limits and the `partial` flag. +package discovery + +import ( + "cmp" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "net/netip" + "slices" + "strconv" + "strings" + "time" + + "github.com/v10networks/nmon/internal/clock" + "github.com/v10networks/nmon/internal/snmp" + "github.com/v10networks/nmon/internal/store" + "github.com/v10networks/nmon/internal/ulid" +) + +// Walk roots. +const ( + oidSystem = "1.3.6.1.2.1.1" + oidIfTable = "1.3.6.1.2.1.2.2.1" + oidIfXTable = "1.3.6.1.2.1.31.1.1.1" +) + +// Target builds the SNMP target for a device's credential. port 0 means 161. +func Target(addr netip.Addr, port int, c *store.SNMPCredential, clk clock.Clock) snmp.Target { + t := snmp.Target{Addr: netip.AddrPortFrom(addr, uint16(port)), Clock: clk} + t.Cred = snmp.Credential{ + Community: c.Community, V3User: c.V3User, V3AuthProto: c.V3AuthProto, V3AuthPass: c.V3AuthPass, + V3PrivProto: c.V3PrivProto, V3PrivPass: c.V3PrivPass, V3Context: c.V3Context, + } + t.Cred.Version = strings.TrimPrefix(c.Version, "v") + return t +} + +// Iface is one interface as the device reports it. +type Iface struct { + IfIndex int64 + Name, Descr string + Alias string + PhysAddr string // colon-separated lower-case hex + Speed int64 + HighSpeed int64 + Type int64 + Admin, Oper int64 + HasHC bool +} + +// Result is what Collect learned. +type Result struct { + SysDescr, SysObjectID, SysName string + SysUpTime time.Duration + Ifaces []Iface + StartedAt, FinishedAt time.Time +} + +// Collect walks the system group and the interface tables over sess, which +// must be a "discovery" session. +func Collect(ctx context.Context, sess *snmp.Session, clk clock.Clock) (*Result, error) { + res := &Result{StartedAt: clk.Now().UTC()} + err := sess.Walk(ctx, oidSystem, 0, func(vb snmp.Varbind) error { + switch vb.OID { + case oidSystem + ".1.0": + res.SysDescr = text(vb) + case oidSystem + ".2.0": + res.SysObjectID, _ = vb.Value.(string) + case oidSystem + ".3.0": + if v, ok := vb.Uint64(); ok { + res.SysUpTime = time.Duration(v) * 10 * time.Millisecond + } + case oidSystem + ".5.0": + res.SysName = text(vb) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("walk system group: %w", err) + } + + ifs := map[int64]*Iface{} + get := func(idx int64) *Iface { + i := ifs[idx] + if i == nil { + i = &Iface{IfIndex: idx} + ifs[idx] = i + } + return i + } + for _, c := range []struct { + name string + root string + set func(*Iface, snmp.Varbind) + }{ + {"ifIndex", oidIfTable + ".1", func(*Iface, snmp.Varbind) {}}, + {"ifDescr", oidIfTable + ".2", func(i *Iface, vb snmp.Varbind) { i.Descr = text(vb) }}, + {"ifType", oidIfTable + ".3", func(i *Iface, vb snmp.Varbind) { i.Type = num(vb) }}, + {"ifSpeed", oidIfTable + ".5", func(i *Iface, vb snmp.Varbind) { i.Speed = num(vb) }}, + {"ifPhysAddress", oidIfTable + ".6", func(i *Iface, vb snmp.Varbind) { i.PhysAddr = mac(vb) }}, + {"ifAdminStatus", oidIfTable + ".7", func(i *Iface, vb snmp.Varbind) { i.Admin = num(vb) }}, + {"ifOperStatus", oidIfTable + ".8", func(i *Iface, vb snmp.Varbind) { i.Oper = num(vb) }}, + {"ifName", oidIfXTable + ".1", func(i *Iface, vb snmp.Varbind) { i.Name = text(vb) }}, + {"ifHighSpeed", oidIfXTable + ".15", func(i *Iface, vb snmp.Varbind) { i.HighSpeed = num(vb) }}, + {"ifAlias", oidIfXTable + ".18", func(i *Iface, vb snmp.Varbind) { i.Alias = text(vb) }}, + {"ifHCInOctets", oidIfXTable + ".6", func(i *Iface, vb snmp.Varbind) { i.HasHC = true }}, + } { + prefix := c.root + "." + err := sess.Walk(ctx, c.root, 0, func(vb snmp.Varbind) error { + idx, err := strconv.ParseInt(strings.TrimPrefix(vb.OID, prefix), 10, 64) + if err != nil { + return nil // not a plain single-index row + } + c.set(get(idx), vb) + return nil + }) + if err != nil { + return nil, fmt.Errorf("walk %s: %w", c.name, err) + } + } + res.Ifaces = make([]Iface, 0, len(ifs)) + for _, i := range ifs { + res.Ifaces = append(res.Ifaces, *i) + } + sortIfaces(res.Ifaces) + res.FinishedAt = clk.Now().UTC() + return res, nil +} + +// Seed discovers dev over sess (a "discovery" session) and records the result: +// sys_* fields, interface rows and a discoveries row. +func Seed(ctx context.Context, st store.Store, sess *snmp.Session, dev *store.Device, clk clock.Clock) error { + res, err := Collect(ctx, sess, clk) + if err != nil { + return err + } + return Apply(ctx, st, dev, res, clk) +} + +// Apply writes a Collect result for dev. Existing interfaces are matched by +// ifName, then by ifDescr plus MAC, and keep their IDs; the rest get new ULIDs. +// Interfaces the device no longer reports are left alone (TODO(M2): retire or review). +func Apply(ctx context.Context, st store.Store, dev *store.Device, res *Result, clk clock.Clock) error { + return st.WithTx(ctx, func(tx store.Store) error { + existing, err := tx.ListInterfaces(ctx, dev.ID, false) + if err != nil { + return err + } + rows := match(existing, res.Ifaces, clk.Now()) + // UpdateDevice writes every mutable column, so start from the stored row + // rather than a caller's copy that may predate SetAddresses. + cur, err := tx.GetDeviceByID(ctx, dev.ID) + if err != nil { + return err + } + cur.SysDescr, cur.SysObjectID, cur.SysName = res.SysDescr, res.SysObjectID, res.SysName + if cur.Profile == "" { + cur.Profile = "generic" + } + if err := tx.UpdateDevice(ctx, cur); err != nil { + return err + } + *dev = *cur + if err := tx.UpsertInterfaces(ctx, dev.ID, rows); err != nil { + return err + } + tables, _ := json.Marshal(map[string]int{"interfaces": len(rows)}) + return tx.RecordDiscovery(ctx, &store.Discovery{ + DeviceID: dev.ID, StartedAt: res.StartedAt, FinishedAt: res.FinishedAt, TablesJSON: string(tables), + }) + }) +} + +// match pairs discovered interfaces with existing rows and returns the rows to upsert. +func match(existing []store.Interface, found []Iface, now time.Time) []store.Interface { + byName := map[string][]int{} + byDescrMAC := map[string][]int{} + for k, e := range existing { + if e.IfName != "" { + byName[e.IfName] = append(byName[e.IfName], k) + } + key := e.IfDescr + "\x00" + e.IfPhysAddr + byDescrMAC[key] = append(byDescrMAC[key], k) + } + used := make([]bool, len(existing)) + take := func(c []int) int { + for _, k := range c { + if !used[k] { + used[k] = true + return k + } + } + return -1 + } + out := make([]store.Interface, 0, len(found)) + for _, f := range found { + k := -1 + if f.Name != "" { + k = take(byName[f.Name]) + } + if k < 0 { + k = take(byDescrMAC[f.Descr+"\x00"+f.PhysAddr]) + } + var row store.Interface + if k >= 0 { + row = existing[k] + } else { + row = store.Interface{ID: ulid.New(now)} + } + row.IfIndex, row.IfName, row.IfDescr, row.IfAlias = f.IfIndex, f.Name, f.Descr, f.Alias + row.IfPhysAddr, row.IfSpeed, row.IfHighSpeed, row.IfType = f.PhysAddr, f.Speed, f.HighSpeed, f.Type + row.AdminStatus, row.OperStatus, row.HasHCCounters = f.Admin, f.Oper, f.HasHC + out = append(out, row) + } + return out +} + +func sortIfaces(s []Iface) { + slices.SortFunc(s, func(a, b Iface) int { return cmp.Compare(a.IfIndex, b.IfIndex) }) +} +func text(vb snmp.Varbind) string { + switch v := vb.Value.(type) { + case []byte: + return strings.ToValidUTF8(string(v), "?") + case string: + return v + } + return "" +} + +func num(vb snmp.Varbind) int64 { + v, _ := vb.Int64() + return v +} + +func mac(vb snmp.Varbind) string { + b, _ := vb.Value.([]byte) + if len(b) == 0 { + return "" + } + h := hex.EncodeToString(b) + parts := make([]string, 0, len(b)) + for i := 0; i < len(h); i += 2 { + parts = append(parts, h[i:i+2]) + } + return strings.Join(parts, ":") +} diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go new file mode 100644 index 0000000..bf59229 --- /dev/null +++ b/internal/discovery/discovery_test.go @@ -0,0 +1,115 @@ +package discovery_test + +import ( + "context" + "strconv" + "strings" + "testing" + "time" + + "github.com/v10networks/nmon/internal/clock" + "github.com/v10networks/nmon/internal/fixturegen" + "github.com/v10networks/nmon/internal/simagent" + "github.com/v10networks/nmon/internal/testutil" + "github.com/v10networks/nmon/internal/ulid" +) + +// shuffled returns rec with every interface index i replaced by 100+n+1-i, in +// both ifTable and ifXTable, so every port keeps its name but changes ifIndex. +func shuffled(rec *simagent.Recording, n int) *simagent.Recording { + const ifT, ifX = "1.3.6.1.2.1.2.2.1.", "1.3.6.1.2.1.31.1.1.1." + remap := func(i int) int { return 100 + n + 1 - i } + var out []simagent.Record + for _, r := range rec.Records() { + for _, p := range []string{ifT, ifX} { + if !strings.HasPrefix(r.OID, p) { + continue + } + cut := strings.LastIndexByte(r.OID, '.') + var i int + for _, c := range r.OID[cut+1:] { + i = i*10 + int(c-'0') + } + r.OID = r.OID[:cut+1] + strconv.Itoa(remap(i)) + if strings.HasPrefix(r.OID, ifT+"1.") { + r.Value = remap(i) + } + } + out = append(out, r) + } + return simagent.NewRecording(out) +} + +func TestSeedCreatesAndKeepsInterfaceIDs(t *testing.T) { + clk := clock.NewFake(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)) + st := testutil.OpenStore(t, clk) + ctx := context.Background() + rec := fixturegen.Generic(fixturegen.Options{Ports: 8, SysName: "sw1", HCCounters: true}) + a := testutil.StartAgent(t, simagent.Config{Recording: rec}) + dev := testutil.AddDevice(t, st, a, "sw1", "public", clk) + + if dev.SysName != "sw1" || dev.SysObjectID != "1.3.6.1.4.1.8072.3.2.10" || dev.SysDescr != "nmon generic fixture" { + t.Fatalf("sys fields: %+v", dev) + } + ifs, err := st.ListInterfaces(ctx, dev.ID, false) + if err != nil || len(ifs) != 8 { + t.Fatalf("interfaces: %d, %v", len(ifs), err) + } + ids := map[string]string{} + for _, i := range ifs { + if !ulid.Valid(i.ID) { + t.Errorf("bad ULID %q", i.ID) + } + if !i.HasHCCounters || i.IfName == "" || i.IfDescr == "" || i.IfSpeed == 0 && i.IfType == 6 { + t.Errorf("incomplete interface %+v", i) + } + ids[i.IfName] = i.ID + } + if ifs[0].IfPhysAddr == "" || strings.Count(ifs[0].IfPhysAddr, ":") != 5 { + t.Errorf("mac %q", ifs[0].IfPhysAddr) + } + d, err := st.LastDiscovery(ctx, dev.ID) + if err != nil || d.Partial || d.TablesJSON == "{}" { + t.Fatalf("discovery row: %+v %v", d, err) + } + + // Same device again with every ifIndex shuffled: IDs must follow the ifName. + a2 := testutil.StartAgent(t, simagent.Config{Recording: shuffled(rec, 8)}) + cred, err := st.GetCredential(ctx, *dev.SNMPCredentialID) + if err != nil { + t.Fatal(err) + } + testutil.Seed(t, st, a2, dev, cred, clk) + after, err := st.ListInterfaces(ctx, dev.ID, false) + if err != nil || len(after) != 8 { + t.Fatalf("after reshuffle: %d, %v", len(after), err) + } + moved := 0 + for _, i := range after { + if ids[i.IfName] != i.ID { + t.Errorf("%s: id %s, was %s", i.IfName, i.ID, ids[i.IfName]) + } + if i.IfIndex > 100 { + moved++ + } + } + if moved != 8 { + t.Errorf("%d interfaces carry the new ifIndex, want 8", moved) + } +} + +func TestSeedWithoutHCCounters(t *testing.T) { + clk := clock.NewFake(time.Now()) + st := testutil.OpenStore(t, clk) + a := testutil.StartAgent(t, simagent.Config{Recording: fixturegen.Generic(fixturegen.Options{Ports: 4})}) + dev := testutil.AddDevice(t, st, a, "old", "public", clk) + ifs, _ := st.ListInterfaces(context.Background(), dev.ID, false) + if len(ifs) != 4 { + t.Fatalf("%d interfaces", len(ifs)) + } + for _, i := range ifs { + if i.HasHCCounters { + t.Errorf("%s: has_hc_counters set", i.IfName) + } + } +} diff --git a/internal/poller/cycle.go b/internal/poller/cycle.go new file mode 100644 index 0000000..5a82341 --- /dev/null +++ b/internal/poller/cycle.go @@ -0,0 +1,299 @@ +package poller + +import ( + "context" + "errors" + "fmt" + "net/netip" + + "github.com/v10networks/nmon/internal/discovery" + "github.com/v10networks/nmon/internal/icmp" + "github.com/v10networks/nmon/internal/metrics" + "github.com/v10networks/nmon/internal/store" +) + +// kind says how a varbind becomes a sample value. +type kind int + +const ( + kindUptime kind = iota // TimeTicks to seconds + kindCounter // unsigned + kindStatus // integer enum +) + +// item is what one requested OID means. +type item struct { + name string // metric name + iface string // interface ULID; empty for device-level series + kind kind +} + +// target is a device as the loop polls it; it is replaced wholesale on reload. +type target struct { + dev store.Device + addr netip.Addr + cred *store.SNMPCredential // nil: the device has no SNMP credential + oids []string + items []item + ifaces int +} + +const oidSysUpTime = "1.3.6.1.2.1.1.3.0" + +// IF-MIB columns polled every cycle: ifTable under 1.3.6.1.2.1.2.2.1, ifXTable under 1.3.6.1.2.1.31.1.1.1. +var ( + hcColumns = []struct { + name, col string + kind kind + }{ + {"ifHCInOctets", "1.3.6.1.2.1.31.1.1.1.6", kindCounter}, + {"ifHCOutOctets", "1.3.6.1.2.1.31.1.1.1.10", kindCounter}, + } + // Without HC counters the 32-bit octet counters are read but keep the + // non-HC names, so a series never silently changes width under one name. + plainColumns = []struct { + name, col string + kind kind + }{ + {"ifInOctets", "1.3.6.1.2.1.2.2.1.10", kindCounter}, + {"ifOutOctets", "1.3.6.1.2.1.2.2.1.16", kindCounter}, + } + commonColumns = []struct { + name, col string + kind kind + }{ + {"ifInErrors", "1.3.6.1.2.1.2.2.1.14", kindCounter}, + {"ifOutErrors", "1.3.6.1.2.1.2.2.1.20", kindCounter}, + {"ifInDiscards", "1.3.6.1.2.1.2.2.1.13", kindCounter}, + {"ifOutDiscards", "1.3.6.1.2.1.2.2.1.19", kindCounter}, + {"ifOperStatus", "1.3.6.1.2.1.2.2.1.8", kindStatus}, + {"ifAdminStatus", "1.3.6.1.2.1.2.2.1.7", kindStatus}, + } +) + +func (p *Poller) buildTarget(ctx context.Context, d *store.Device) (*target, error) { + t := &target{dev: *d} + if a, err := netip.ParseAddr(d.MgmtIP); err == nil { + t.addr = a + } + if d.SNMPCredentialID != nil { + c, err := p.o.Store.GetCredential(ctx, *d.SNMPCredentialID) + if err != nil { + return nil, fmt.Errorf("credential: %w", err) + } + t.cred = c + ifs, err := p.o.Store.ListInterfaces(ctx, d.ID, false) + if err != nil { + return nil, fmt.Errorf("interfaces: %w", err) + } + t.oids = append(t.oids, oidSysUpTime) + t.items = append(t.items, item{name: "sysUpTime", kind: kindUptime}) + for _, i := range ifs { + if i.Excluded || i.RetiredAt != nil { + continue + } + t.ifaces++ + octets := plainColumns + if i.HasHCCounters { + octets = hcColumns + } + for _, cols := range [][]struct { + name, col string + kind kind + }{octets, commonColumns} { + for _, c := range cols { + t.oids = append(t.oids, fmt.Sprintf("%s.%d", c.col, i.IfIndex)) + t.items = append(t.items, item{name: c.name, iface: i.ID, kind: c.kind}) + } + } + } + } + return t, nil +} + +// runSNMP polls the device every interval, starting at a stable per-device offset. +func (l *devLoop) runSNMP(ctx context.Context) { + p := l.p + off := offset + if p.o.Offset != nil { + off = p.o.Offset + } + next := p.clk.Now().Add(off(l.slug, p.o.Interval)) + for sleepUntil(ctx, p.clk, next) { + l.cycle(ctx) + next = advance(next, p.clk.Now(), p.o.Interval) + } +} + +// cycle runs one poll under the cycle deadline. It returns only when the poll +// has finished or been abandoned, so cycles of one device never overlap. +func (l *devLoop) cycle(ctx context.Context) { + p := l.p + t := l.tgt.Load() + if t == nil || t.cred == nil || !t.addr.IsValid() { + return + } + start := p.clk.Now() + cctx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + deadline := p.clk.After(p.o.Interval / 2) // registered before polling starts + go func() { + select { + case <-deadline: + cancel(errDeadline) + case <-cctx.Done(): + } + }() + + samples, skipped, err := l.poll(cctx, t) + if ctx.Err() != nil { + return // shutting down + } + res := CycleResult{Device: l.slug, Err: err, Samples: len(samples), Skipped: skipped} + if err != nil && errors.Is(context.Cause(cctx), errDeadline) { + res.Missed = true + l.missed.Inc() + } + now := p.clk.Now() + l.duration.Set(now.Sub(start).Seconds()) + l.skipped.Add(float64(skipped)) + up := 0.0 + if err == nil { + up = 1 + if terr := p.o.Store.TouchDevice(ctx, t.dev.ID, now.UTC()); terr != nil { + p.log.Warn("touch device", "device", l.slug, "err", terr) + } + } + samples = append(samples, metrics.New("nmon_snmp_up", up, now, metrics.LabelDevice, l.slug)) + if werr := p.o.Writer.Write(ctx, samples); werr != nil { + p.log.Warn("write samples", "device", l.slug, "err", werr) + } + if l.snmpUp == nil || *l.snmpUp != (err == nil) { + ok := err == nil + l.snmpUp = &ok + if ok { + p.log.Info("snmp up", "device", l.slug) + } else { + p.log.Warn("snmp down", "device", l.slug, "err", err, "deadline_missed", res.Missed) + } + } + if p.o.OnCycle != nil { + p.o.OnCycle(res) + } +} + +// poll does one multi-varbind GET and converts the answer to samples. +func (l *devLoop) poll(ctx context.Context, t *target) ([]metrics.Sample, int, error) { + p := l.p + select { + case p.sem <- struct{}{}: + defer func() { <-p.sem }() + case <-ctx.Done(): + return nil, 0, ctx.Err() + } + tg := discovery.Target(t.addr, p.o.SNMPPort, t.cred, p.clk) + sess, err := p.o.Dial(ctx, tg) + if err != nil { + return nil, 0, err + } + defer sess.Close() + vbs, err := sess.Get(ctx, t.oids) + if err != nil { + return nil, 0, err + } + if len(vbs) != len(t.oids) { + return nil, 0, fmt.Errorf("asked for %d varbinds, got %d", len(t.oids), len(vbs)) + } + ts := p.clk.Now() + out := make([]metrics.Sample, 0, len(vbs)+1) + skipped := 0 + for i, vb := range vbs { + it := t.items[i] + if vb.Exception() || vb.OID != t.oids[i] { + skipped++ + continue + } + var v float64 + switch it.kind { + case kindUptime: + u, ok := vb.Uint64() + if !ok { + skipped++ + continue + } + v = float64(u) / 100 + case kindCounter: + u, ok := vb.Uint64() + if !ok { + skipped++ + continue + } + v = float64(u) + case kindStatus: + n, ok := vb.Int64() + if !ok { + skipped++ + continue + } + v = float64(n) + } + if it.iface == "" { + out = append(out, metrics.New(it.name, v, ts, metrics.LabelDevice, l.slug)) + } else { + out = append(out, metrics.New(it.name, v, ts, metrics.LabelDevice, l.slug, metrics.LabelIface, it.iface)) + } + } + return out, skipped, nil +} + +// runICMP probes the preferred management address every 15 s. An unsupported +// probe writes nothing and stops: it is unknown, never down. +func (l *devLoop) runICMP(ctx context.Context) { + p := l.p + next := p.clk.Now().Add(offset(l.slug, icmpInterval)) + var ( + known, up bool + hits, misses int + ) + for sleepUntil(ctx, p.clk, next) { + next = advance(next, p.clk.Now(), icmpInterval) + t := l.tgt.Load() + if t == nil || !t.addr.IsValid() { + continue + } + rtt, err := p.o.Pinger.Ping(ctx, t.addr, icmpTimeout) + if ctx.Err() != nil { + return + } + if errors.Is(err, icmp.ErrUnsupported) { + p.log.Warn("icmp probe unsupported; not probing", "device", l.slug, "err", err) + return + } + now := p.clk.Now() + var out []metrics.Sample + if err == nil { + hits, misses = hits+1, 0 + if !known || (!up && hits >= icmpUpAfter) { + known, up = true, true + } + out = append(out, metrics.New("nmon_rtt_seconds", rtt.Seconds(), now, metrics.LabelDevice, l.slug)) + } else { + hits, misses = 0, misses+1 + if misses >= icmpDownAfter && (!known || up) { + known, up = true, false + } + } + if known { + v := 0.0 + if up { + v = 1 + } + out = append(out, metrics.New("nmon_icmp_up", v, now, metrics.LabelDevice, l.slug)) + } + if len(out) > 0 { + if werr := p.o.Writer.Write(ctx, out); werr != nil { + p.log.Warn("write icmp samples", "device", l.slug, "err", werr) + } + } + } +} diff --git a/internal/poller/poller.go b/internal/poller/poller.go new file mode 100644 index 0000000..0755c23 --- /dev/null +++ b/internal/poller/poller.go @@ -0,0 +1,276 @@ +// Package poller runs the 60 s collection path: a jittered, deadline-bounded +// SNMP GET of the generic interface counters per device, and an ICMP probe +// every 15 s. Each device has exactly one SNMP goroutine, so a device is never +// polled by two cycles at once; a global semaphore caps in-flight polls. +// +// TODO(M2): profiles instead of the hard-coded metric list, varbind +// adaptation on tooBig, backoff and the degraded state, the 30 s cycle for +// devices without HC counters. +package poller + +import ( + "context" + "errors" + "hash/fnv" + "log/slog" + "net/netip" + "sync" + "sync/atomic" + "time" + + "github.com/v10networks/nmon/internal/clock" + "github.com/v10networks/nmon/internal/icmp" + "github.com/v10networks/nmon/internal/metrics" + "github.com/v10networks/nmon/internal/selfmon" + "github.com/v10networks/nmon/internal/snmp" + "github.com/v10networks/nmon/internal/store" +) + +// Defaults. +const ( + DefaultInterval = 60 * time.Second + DefaultMaxInflight = 512 + icmpInterval = 15 * time.Second + selfmonInterval = 15 * time.Second + icmpTimeout = 2 * time.Second + icmpDownAfter = 3 // consecutive misses + icmpUpAfter = 2 // consecutive hits after being down +) + +// Session is the part of *snmp.Session the poller uses. +type Session interface { + Get(ctx context.Context, oids []string) ([]snmp.Varbind, error) + Close() error +} + +// Dialer opens a poll-path session. The default is snmp.Dial with path "poll". +type Dialer func(ctx context.Context, t snmp.Target) (Session, error) + +// Pinger sends one echo request. *icmp.Pinger implements it. +type Pinger interface { + Ping(ctx context.Context, addr netip.Addr, timeout time.Duration) (time.Duration, error) +} + +// CycleResult describes one finished SNMP cycle of one device. +type CycleResult struct { + Device string + Err error // nil on success + Missed bool // the cycle deadline passed before the poll finished + Samples int // interface and sysUpTime samples written + Skipped int // varbinds skipped: exceptions or non-numeric values +} + +// Options configures a Poller. Store and Writer are required. +type Options struct { + Store store.Store + Writer metrics.Writer // the hot write queue + Clock clock.Clock // default clock.Real() + + Interval time.Duration // per-device SNMP interval; default 60 s. The cycle deadline is half of it + MaxInflight int // global cap on concurrent polls; default 512 + SNMPPort int // dev/test override of UDP 161; 0 = 161 + + Registry *selfmon.Registry // default selfmon.Default + Logger *slog.Logger // default slog.Default() + + // Test hooks. + Offset func(slug string, interval time.Duration) time.Duration // first-poll offset; default stable hash + Dial Dialer + Pinger Pinger + OnCycle func(CycleResult) +} + +// Poller schedules and runs all device polls. +type Poller struct { + o Options + clk clock.Clock + log *slog.Logger + reg *selfmon.Registry + sem chan struct{} + refresh chan struct{} + started chan struct{} + once sync.Once + + loops map[string]*devLoop // owned by Run + wg sync.WaitGroup +} + +// New returns a Poller; nothing runs until Run. +func New(o Options) *Poller { + if o.Clock == nil { + o.Clock = clock.Real() + } + if o.Interval <= 0 { + o.Interval = DefaultInterval + } + if o.MaxInflight <= 0 { + o.MaxInflight = DefaultMaxInflight + } + if o.Registry == nil { + o.Registry = selfmon.Default + } + if o.Logger == nil { + o.Logger = slog.Default() + } + if o.Dial == nil { + o.Dial = func(ctx context.Context, t snmp.Target) (Session, error) { + s, err := snmp.Dial(ctx, t, "poll") + if err != nil { + return nil, err + } + return s, nil + } + } + if o.Pinger == nil { + o.Pinger = icmp.New(o.Clock) + } + return &Poller{ + o: o, clk: o.Clock, log: o.Logger.With("component", "poller"), reg: o.Registry, + sem: make(chan struct{}, o.MaxInflight), + refresh: make(chan struct{}, 1), + started: make(chan struct{}), + loops: map[string]*devLoop{}, + } +} + +// Refresh asks the poller to reload its device list from the store. It never blocks. +func (p *Poller) Refresh() { + select { + case p.refresh <- struct{}{}: + default: + } +} + +// Started is closed once the first device list has been loaded and its loops started. +func (p *Poller) Started() <-chan struct{} { return p.started } + +// Run polls until ctx ends, then waits for every loop to stop. +func (p *Poller) Run(ctx context.Context) error { + defer p.wg.Wait() + if err := p.reload(ctx); err != nil { + p.log.Error("load devices", "err", err) + } + p.once.Do(func() { close(p.started) }) + tick := p.clk.NewTicker(selfmonInterval) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-p.refresh: + if err := p.reload(ctx); err != nil { + p.log.Error("reload devices", "err", err) + } + case <-tick.C(): + now := p.clk.Now() + if err := p.o.Writer.Write(ctx, p.reg.Snapshot(now)); err != nil { + p.log.Warn("write nmon_ metrics", "err", err) + } + } + } +} + +// reload syncs the device loops with the store. +func (p *Poller) reload(ctx context.Context) error { + var devs []store.Device + for off := int64(0); ; { + page, total, err := p.o.Store.ListDevices(ctx, store.DeviceFilter{}, store.Page{Offset: off, Limit: store.MaxLimit}) + if err != nil { + return err + } + devs = append(devs, page...) + off += int64(len(page)) + if len(page) == 0 || off >= total { + break + } + } + seen := make(map[string]bool, len(devs)) + for i := range devs { + d := &devs[i] + if !d.Enabled || d.RetiredAt != nil { + continue + } + t, err := p.buildTarget(ctx, d) + if err != nil { + p.log.Error("load device", "device", d.Slug, "err", err) + if l := p.loops[d.Slug]; l != nil { + seen[d.Slug] = true // keep polling with the previous settings + } + continue + } + seen[d.Slug] = true + l := p.loops[d.Slug] + if l == nil { + l = p.startLoop(ctx, d.Slug) + } + l.tgt.Store(t) + } + for slug, l := range p.loops { + if !seen[slug] { + l.cancel() + delete(p.loops, slug) + } + } + return nil +} + +func (p *Poller) startLoop(ctx context.Context, slug string) *devLoop { + lctx, cancel := context.WithCancel(ctx) + l := &devLoop{ + p: p, slug: slug, cancel: cancel, + duration: p.reg.Gauge("nmon_poll_duration_seconds", "device", slug), + missed: p.reg.Counter("nmon_poll_deadline_missed_total", "device", slug), + skipped: p.reg.Counter("nmon_poll_skipped_varbinds_total", "device", slug), + } + p.loops[slug] = l + p.wg.Add(2) + go func() { defer p.wg.Done(); l.runSNMP(lctx) }() + go func() { defer p.wg.Done(); l.runICMP(lctx) }() + return l +} + +// offset spreads first polls uniformly over the interval, stably per device. +func offset(slug string, interval time.Duration) time.Duration { + h := fnv.New64a() + h.Write([]byte(slug)) + return time.Millisecond + time.Duration(h.Sum64()%uint64(interval-time.Millisecond)) +} + +// sleepUntil waits for the clock to reach t; it returns false if ctx ended first. +func sleepUntil(ctx context.Context, clk clock.Clock, t time.Time) bool { + d := t.Sub(clk.Now()) + if d <= 0 { + return ctx.Err() == nil + } + select { + case <-ctx.Done(): + return false + case <-clk.After(d): + return true + } +} + +// advance returns the first scheduled time after now. +func advance(next, now time.Time, every time.Duration) time.Time { + next = next.Add(every) + for !next.After(now) { + next = next.Add(every) + } + return next +} + +var errDeadline = errors.New("poller: cycle deadline") + +// devLoop is the pair of goroutines serving one device. +type devLoop struct { + p *Poller + slug string + cancel context.CancelFunc + tgt atomic.Pointer[target] + + duration selfmon.Gauge + missed selfmon.Counter + skipped selfmon.Counter + + snmpUp *bool // last state logged; touched only by the SNMP goroutine +} diff --git a/internal/poller/poller_test.go b/internal/poller/poller_test.go new file mode 100644 index 0000000..b46fc65 --- /dev/null +++ b/internal/poller/poller_test.go @@ -0,0 +1,604 @@ +package poller_test + +import ( + "context" + "errors" + "net/netip" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gosnmp/gosnmp" + + "github.com/v10networks/nmon/internal/clock" + "github.com/v10networks/nmon/internal/fixturegen" + "github.com/v10networks/nmon/internal/icmp" + "github.com/v10networks/nmon/internal/metrics" + "github.com/v10networks/nmon/internal/poller" + "github.com/v10networks/nmon/internal/selfmon" + "github.com/v10networks/nmon/internal/simagent" + "github.com/v10networks/nmon/internal/snmp" + "github.com/v10networks/nmon/internal/store" + "github.com/v10networks/nmon/internal/testutil" + "github.com/v10networks/nmon/internal/ulid" +) + +const interval = 60 * time.Second + +var t0 = time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + +type writer struct { + mu sync.Mutex + all []metrics.Sample +} + +func (w *writer) Write(_ context.Context, s []metrics.Sample) error { + w.mu.Lock() + w.all = append(w.all, s...) + w.mu.Unlock() + return nil +} + +func (w *writer) samples(name string) []metrics.Sample { + w.mu.Lock() + defer w.mu.Unlock() + var out []metrics.Sample + for _, s := range w.all { + if s.Name == name { + out = append(out, s) + } + } + return out +} + +func (w *writer) snapshot() []metrics.Sample { + w.mu.Lock() + defer w.mu.Unlock() + return append([]metrics.Sample(nil), w.all...) +} + +type pinger struct { + mu sync.Mutex + results []error // consumed in order; the last one repeats + calls int +} + +func (p *pinger) Ping(context.Context, netip.Addr, time.Duration) (time.Duration, error) { + p.mu.Lock() + defer p.mu.Unlock() + var err error + if len(p.results) > 0 { + err = p.results[min(p.calls, len(p.results)-1)] + } + p.calls++ + return 3 * time.Millisecond, err +} + +func (p *pinger) count() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls } + +type harness struct { + t *testing.T + clk *clock.Fake + st store.Store + w *writer + p *poller.Poller + cycles chan poller.CycleResult + ping *pinger + cancel context.CancelFunc + done chan struct{} + counts map[string]int +} + +type opts struct { + maxInflight int + dial poller.Dialer + offset func(string, time.Duration) time.Duration +} + +func newHarness(t *testing.T, clk *clock.Fake, st store.Store, o opts) *harness { + h := &harness{t: t, clk: clk, st: st, w: &writer{}, cycles: make(chan poller.CycleResult, 4096), ping: &pinger{}, counts: map[string]int{}} + h.p = poller.New(poller.Options{ + Store: st, Writer: h.w, Clock: clk, Interval: interval, MaxInflight: o.maxInflight, + Dial: o.dial, Pinger: h.ping, Offset: o.offset, + OnCycle: func(r poller.CycleResult) { h.cycles <- r }, + }) + return h +} + +// start runs the poller and waits until every loop is blocked on the clock. +func (h *harness) start(devices int) { + ctx, cancel := context.WithCancel(context.Background()) + h.cancel = cancel + h.done = make(chan struct{}) + go func() { h.p.Run(ctx); close(h.done) }() + h.t.Cleanup(h.stop) + <-h.p.Started() + // per device: SNMP and ICMP sleeps; plus the nmon_ metrics ticker + eventually(h.t, func() bool { return h.clk.Waiters() >= 2*devices+1 }) +} + +func (h *harness) stop() { + h.cancel() + <-h.done +} + +// drain collects finished cycles for up to wait. +func (h *harness) drain(wait time.Duration) []poller.CycleResult { + var out []poller.CycleResult + for { + select { + case r := <-h.cycles: + h.counts[r.Device]++ + out = append(out, r) + case <-time.After(wait): + return out + } + } +} + +// until advances the fake clock in quarter-interval steps until cond holds. +func (h *harness) until(cond func() bool) { + h.t.Helper() + for i := 0; i < 40 && !cond(); i++ { + h.clk.Advance(interval / 4) + h.drain(25 * time.Millisecond) + } + if !cond() { + h.t.Fatal("condition not reached after advancing the clock") + } +} + +func eventually(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("timed out waiting for condition") + } + time.Sleep(2 * time.Millisecond) + } +} + +// fixedOffset starts every device 1 s into the interval, so all cycles share one deadline window. +func fixedOffset(string, time.Duration) time.Duration { return time.Second } + +func counter(name string, kv ...string) float64 { + return selfmon.Default.Counter(name, kv...).Value() +} + +func TestPollWritesExactLabelsAndRisingValues(t *testing.T) { + clk := clock.NewFake(t0) + st := testutil.OpenStore(t, clk) + a := testutil.StartAgent(t, simagent.Config{ + Recording: fixturegen.Generic(fixturegen.Options{Ports: 8, HCCounters: true, Counters: true}), Clock: clk, + }) + dev := testutil.AddDevice(t, st, a, "sw1", "public", clk) + ifs, _ := st.ListInterfaces(context.Background(), dev.ID, false) + ids := map[string]bool{} + for _, i := range ifs { + ids[i.ID] = true + } + a.ResetStats() + pdus := func(typ string) float64 { return counter("nmon_pdus_total", "path", "poll", "type", typ) } + get0, next0, bulk0 := pdus("get"), pdus("getnext"), pdus("getbulk") + + h := newHarness(t, clk, st, opts{}) + h.p = poller.New(poller.Options{ + Store: st, Writer: h.w, Clock: clk, Interval: interval, Pinger: h.ping, SNMPPort: int(a.Addr().Port()), + OnCycle: func(r poller.CycleResult) { h.cycles <- r }, + }) + h.start(1) + h.until(func() bool { return h.counts[dev.Slug] >= 3 }) + time.Sleep(100 * time.Millisecond) // let any cycle in flight finish; the clock no longer moves + h.drain(10 * time.Millisecond) + n := h.counts[dev.Slug] + + names := map[string]bool{} + perIface := map[string][]float64{} + for _, s := range h.w.snapshot() { + if len(s.Name) < 3 || s.Name[:3] == "nmo" && s.Name != "nmon_snmp_up" { + continue // selfmon snapshots and icmp are checked elsewhere + } + names[s.Name] = true + switch s.Name { + case "sysUpTime", "nmon_snmp_up": + if len(s.Labels) != 1 || s.Labels[0] != (metrics.Label{Name: "device", Value: dev.Slug}) { + t.Fatalf("%s labels %v", s.Name, s.Labels) + } + if s.Name == "nmon_snmp_up" && s.Value != 1 { + t.Fatalf("snmp_up %v", s.Value) + } + default: + if len(s.Labels) != 2 || s.Labels[0].Name != "device" || s.Labels[0].Value != dev.Slug || + s.Labels[1].Name != "iface" || !ids[s.Labels[1].Value] { + t.Fatalf("%s labels %v", s.Name, s.Labels) + } + if s.Name == "ifHCInOctets" { + perIface[s.Labels[1].Value] = append(perIface[s.Labels[1].Value], s.Value) + } + } + } + want := []string{"sysUpTime", "nmon_snmp_up", "ifHCInOctets", "ifHCOutOctets", "ifInErrors", "ifOutErrors", + "ifInDiscards", "ifOutDiscards", "ifOperStatus", "ifAdminStatus"} + if len(names) != len(want) { + t.Errorf("metric names %v", names) + } + for _, w := range want { + if !names[w] { + t.Errorf("missing %s", w) + } + } + if got := len(h.w.samples("ifHCInOctets")); got != 8*n { + t.Errorf("%d ifHCInOctets samples for %d cycles of 8 ports", got, n) + } + for id, vs := range perIface { + for i := 1; i < len(vs); i++ { + if vs[i] <= vs[i-1] { + t.Errorf("iface %s: %v not rising", id, vs) + } + } + } + if up := h.w.samples("sysUpTime"); len(up) != n || up[1].Value <= up[0].Value { + t.Errorf("sysUpTime %v", up) + } + + // 1 sysUpTime + 8 ports x 8 objects = 65 varbinds = 2 PDUs of at most 40, all GET. + st1 := a.Stats() + if st1.Get != uint64(2*n) || st1.GetNext != 0 || st1.GetBulk != 0 { + t.Errorf("agent saw %+v for %d cycles, want %d GETs and no walks", st1, n, 2*n) + } + if d := pdus("get") - get0; d != float64(2*n) { + t.Errorf("nmon_pdus_total{poll,get} rose by %v, want %d", d, 2*n) + } + if pdus("getnext") != next0 || pdus("getbulk") != bulk0 { + t.Error("walk PDUs counted on the poll path") + } + if got, _ := st.GetDeviceBySlug(context.Background(), dev.Slug); got.LastSeen == nil { + t.Error("last_seen not updated") + } +} + +func TestPollWithoutHCUsesPlainNames(t *testing.T) { + clk := clock.NewFake(t0) + st := testutil.OpenStore(t, clk) + a := testutil.StartAgent(t, simagent.Config{Recording: fixturegen.Generic(fixturegen.Options{Ports: 4, Counters: true}), Clock: clk}) + dev := testutil.AddDevice(t, st, a, "old", "public", clk) + h := newHarness(t, clk, st, opts{}) + h.p = poller.New(poller.Options{Store: st, Writer: h.w, Clock: clk, Interval: interval, Pinger: h.ping, + SNMPPort: int(a.Addr().Port()), OnCycle: func(r poller.CycleResult) { h.cycles <- r }}) + h.start(1) + h.until(func() bool { return h.counts[dev.Slug] >= 1 }) + if len(h.w.samples("ifInOctets")) != 4 || len(h.w.samples("ifOutOctets")) != 4 { + t.Errorf("want 4 ifInOctets and ifOutOctets, got %d/%d", len(h.w.samples("ifInOctets")), len(h.w.samples("ifOutOctets"))) + } + if len(h.w.samples("ifHCInOctets")) != 0 || len(h.w.samples("ifHCOutOctets")) != 0 { + t.Error("HC names used for a device without HC counters") + } +} + +// fakeDevices creates n SNMP devices with two interfaces each, no agent behind them. +func fakeDevices(t *testing.T, st store.Store, n int) []*store.Device { + ctx := context.Background() + var out []*store.Device + for i := 0; i < n; i++ { + name := "dev" + string(rune('a'+i)) + cred := &store.SNMPCredential{Name: name, Version: "v2c", Community: name} + if err := st.CreateCredential(ctx, cred); err != nil { + t.Fatal(err) + } + d := &store.Device{Name: name, Kind: store.KindNetwork, SNMPCredentialID: &cred.ID, Enabled: true, Profile: "generic"} + if err := st.CreateDevice(ctx, d); err != nil { + t.Fatal(err) + } + if err := st.SetAddresses(ctx, d.ID, []string{"::1"}); err != nil { + t.Fatal(err) + } + var ifs []store.Interface + for j := 1; j <= 2; j++ { + ifs = append(ifs, store.Interface{ID: ulid.New(t0), IfIndex: int64(j), IfName: "p" + string(rune('0'+j)), HasHCCounters: true}) + } + if err := st.UpsertInterfaces(ctx, d.ID, ifs); err != nil { + t.Fatal(err) + } + out = append(out, d) + } + return out +} + +// blockingSession answers every Get with ones after release, or fails with ctx.Err(). +type blocker struct { + mu sync.Mutex + cur map[string]int + max map[string]int + global atomic.Int32 + gmax atomic.Int32 + release chan struct{} + dialed atomic.Int32 +} + +func newBlocker() *blocker { + return &blocker{cur: map[string]int{}, max: map[string]int{}, release: make(chan struct{})} +} + +type blockSession struct { + b *blocker + comm string +} + +func (b *blocker) dial(_ context.Context, tg snmp.Target) (poller.Session, error) { + b.dialed.Add(1) + return &blockSession{b, tg.Cred.Community}, nil +} + +func (s *blockSession) Get(ctx context.Context, oids []string) ([]snmp.Varbind, error) { + b := s.b + b.mu.Lock() + b.cur[s.comm]++ + b.max[s.comm] = max(b.max[s.comm], b.cur[s.comm]) + b.mu.Unlock() + g := b.global.Add(1) + for { + old := b.gmax.Load() + if g <= old || b.gmax.CompareAndSwap(old, g) { + break + } + } + defer func() { + b.global.Add(-1) + b.mu.Lock() + b.cur[s.comm]-- + b.mu.Unlock() + }() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-b.release: + } + out := make([]snmp.Varbind, len(oids)) + for i, o := range oids { + out[i] = snmp.Varbind{OID: o, Type: gosnmp.Integer, Value: 1} + } + return out, nil +} + +func (s *blockSession) Close() error { return nil } + +func TestPerDeviceConcurrencyIsOneEvenWhenDeadlineBeatsLatency(t *testing.T) { + clk := clock.NewFake(t0) + st := testutil.OpenStore(t, clk) + devs := fakeDevices(t, st, 3) + b := newBlocker() + missed0 := map[string]float64{} + for _, d := range devs { + missed0[d.Slug] = counter("nmon_poll_deadline_missed_total", "device", d.Slug) + } + h := newHarness(t, clk, st, opts{dial: b.dial}) + h.start(3) + h.until(func() bool { + for _, d := range devs { + if h.counts[d.Slug] < 3 { + return false + } + } + return true + }) + b.mu.Lock() + defer b.mu.Unlock() + for _, d := range devs { + if m := b.max[d.Name]; m != 1 { + t.Errorf("%s: max concurrent polls %d, want 1", d.Slug, m) + } + got := counter("nmon_poll_deadline_missed_total", "device", d.Slug) - missed0[d.Slug] + if got < 3 || got != float64(h.counts[d.Slug]) { + t.Errorf("%s: deadline missed %v times in %d cycles", d.Slug, got, h.counts[d.Slug]) + } + if dur := selfmon.Default.Gauge("nmon_poll_duration_seconds", "device", d.Slug).Value(); dur < 30 { + t.Errorf("%s: poll duration %vs, want at least the 30s deadline", d.Slug, dur) + } + } + for _, s := range h.w.samples("nmon_snmp_up") { + if s.Value != 0 { + t.Fatalf("snmp_up %v after a missed deadline", s.Value) + } + } +} + +func TestGlobalInflightCap(t *testing.T) { + clk := clock.NewFake(t0) + st := testutil.OpenStore(t, clk) + devs := fakeDevices(t, st, 5) + b := newBlocker() + h := newHarness(t, clk, st, opts{dial: b.dial, maxInflight: 2, offset: fixedOffset}) + h.start(5) + clk.Advance(2 * time.Second) // every device starts its cycle + eventually(t, func() bool { return b.global.Load() == 2 }) + time.Sleep(100 * time.Millisecond) + if g := b.gmax.Load(); g != 2 { + t.Fatalf("max in flight %d with --max-inflight 2", g) + } + close(b.release) + got := map[string]bool{} + deadline := time.After(10 * time.Second) + for len(got) < len(devs) { + select { + case r := <-h.cycles: + if r.Err != nil { + t.Fatalf("%s: %v", r.Device, r.Err) + } + got[r.Device] = true + case <-deadline: + t.Fatalf("only %d of %d devices finished", len(got), len(devs)) + } + } + if g := b.gmax.Load(); g != 2 { + t.Fatalf("max in flight %d after release", g) + } +} + +func TestSlowDeviceDoesNotDelayOthers(t *testing.T) { + clk := clock.NewFake(t0) + st := testutil.OpenStore(t, clk) + rec := fixturegen.Generic(fixturegen.Options{Ports: 8, HCCounters: true, Counters: true}) + ports := map[string]uint16{} + normal := map[string]bool{} + for i := 0; i < 10; i++ { + name := "norm" + string(rune('a'+i)) + a := testutil.StartAgent(t, simagent.Config{Recording: rec, Clock: clk, Community: name}) + d := testutil.AddDevice(t, st, a, name, name, clk) + ports[name] = a.Addr().Port() + normal[d.Slug] = true + } + // The slow device is seeded against a fast agent, then replaced by one that answers after 3 s. + fast := testutil.StartAgent(t, simagent.Config{Recording: rec, Clock: clk, Community: "slow"}) + slow := testutil.AddDevice(t, st, fast, "slow", "slow", clk) + fast.Close() + s := testutil.StartAgent(t, simagent.Config{Recording: rec, Community: "slow", Behaviour: simagent.Behaviour{Latency: 3 * time.Second}}) + ports["slow"] = s.Addr().Port() + + var slowDialed atomic.Bool + dial := func(ctx context.Context, tg snmp.Target) (poller.Session, error) { + tg.Addr = netip.AddrPortFrom(tg.Addr.Addr(), ports[tg.Cred.Community]) + if tg.Cred.Community == "slow" { + defer slowDialed.Store(true) + } + sess, err := snmp.Dial(ctx, tg, "poll") + if err != nil { + return nil, err + } + return sess, nil + } + missed0 := counter("nmon_poll_deadline_missed_total", "device", slow.Slug) + h := newHarness(t, clk, st, opts{dial: dial, offset: fixedOffset}) + h.start(11) + + clk.Advance(2 * time.Second) // one tick: every device starts + done := map[string]bool{} + deadline := time.After(10 * time.Second) + for len(done) < len(normal) { + select { + case r := <-h.cycles: + if r.Device == slow.Slug { + t.Fatalf("slow device finished early: %+v", r) + } + if r.Err != nil { + t.Fatalf("%s: %v", r.Device, r.Err) + } + done[r.Device] = true + case <-deadline: + t.Fatalf("only %d of %d normal devices finished within the first tick", len(done), len(normal)) + } + } + eventually(t, slowDialed.Load) + if got := counter("nmon_poll_deadline_missed_total", "device", slow.Slug); got != missed0 { + t.Fatalf("deadline counted before it passed: %v", got) + } + clk.Advance(interval / 2) // the slow device's cycle deadline + for { + select { + case r := <-h.cycles: + if r.Device != slow.Slug { + continue // normal devices starting their next cycle + } + if !r.Missed || r.Err == nil { + t.Fatalf("slow device result %+v, want a missed deadline", r) + } + if got := counter("nmon_poll_deadline_missed_total", "device", slow.Slug) - missed0; got != 1 { + t.Fatalf("deadline missed counted %v times", got) + } + return + case <-time.After(10 * time.Second): + t.Fatal("slow device cycle never ended") + } + } +} + +func TestICMPStateMachine(t *testing.T) { + clk := clock.NewFake(t0) + st := testutil.OpenStore(t, clk) + d := &store.Device{Name: "vm1", Kind: store.KindVM, Enabled: true, Profile: "generic"} + if err := st.CreateDevice(context.Background(), d); err != nil { + t.Fatal(err) + } + if err := st.SetAddresses(context.Background(), d.ID, []string{"192.0.2.1", "2001:db8::1"}); err != nil { + t.Fatal(err) + } + miss, hit := errors.New("timeout"), error(nil) + h := newHarness(t, clk, st, opts{}) + h.ping.results = []error{hit, miss, miss, miss, hit, hit} + h.start(1) + + var ups []float64 + step := func() { + before := h.ping.count() + for h.ping.count() == before { + clk.Advance(15 * time.Second) + time.Sleep(2 * time.Millisecond) + } + time.Sleep(20 * time.Millisecond) // the write follows the ping + } + for i := 0; i < 6; i++ { + step() + } + for _, s := range h.w.samples("nmon_icmp_up") { + ups = append(ups, s.Value) + } + // hit: up; two misses: nothing changes; third miss: down; first hit: still down; second hit: up. + want := []float64{1, 1, 1, 0, 0, 1} + if len(ups) != len(want) { + t.Fatalf("icmp_up series %v, want %v", ups, want) + } + for i := range want { + if ups[i] != want[i] { + t.Fatalf("icmp_up series %v, want %v", ups, want) + } + } + if n := len(h.w.samples("nmon_rtt_seconds")); n != 3 { + t.Errorf("%d rtt samples, want one per hit", n) + } + for _, s := range h.w.samples("nmon_icmp_up") { + if len(s.Labels) != 1 || s.Labels[0].Value != d.Slug { + t.Fatalf("labels %v", s.Labels) + } + } +} + +func TestICMPUnsupportedWritesNothing(t *testing.T) { + clk := clock.NewFake(t0) + st := testutil.OpenStore(t, clk) + fakeDevices(t, st, 1) + h := newHarness(t, clk, st, opts{dial: newBlocker().dial}) + h.ping.results = []error{icmp.ErrUnsupported} + h.start(1) + for h.ping.count() == 0 { + clk.Advance(15 * time.Second) + time.Sleep(2 * time.Millisecond) + } + for i := 0; i < 5; i++ { + clk.Advance(15 * time.Second) + time.Sleep(2 * time.Millisecond) + } + if n := h.ping.count(); n != 1 { + t.Errorf("pinged %d times after ErrUnsupported", n) + } + if len(h.w.samples("nmon_icmp_up")) != 0 || len(h.w.samples("nmon_rtt_seconds")) != 0 { + t.Error("unsupported probe wrote samples") + } +} + +func TestRefreshPicksUpNewDeviceAndSelfmonIsWritten(t *testing.T) { + clk := clock.NewFake(t0) + st := testutil.OpenStore(t, clk) + b := newBlocker() + close(b.release) + h := newHarness(t, clk, st, opts{dial: b.dial}) + h.start(0) + devs := fakeDevices(t, st, 1) + h.p.Refresh() + eventually(t, func() bool { return clk.Waiters() >= 3 }) + h.until(func() bool { return h.counts[devs[0].Slug] >= 1 }) + eventually(t, func() bool { return len(h.w.samples("nmon_poll_duration_seconds")) > 0 }) + if len(h.w.samples("nmon_pdus_total")) == 0 && len(h.w.samples("nmon_poll_duration_seconds")) == 0 { + t.Error("selfmon snapshot not written") + } +} diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go new file mode 100644 index 0000000..52a7232 --- /dev/null +++ b/internal/testutil/testutil.go @@ -0,0 +1,93 @@ +// Package testutil builds the fixtures WP5 tests share: a migrated SQLite +// store, simagent devices and inventory rows seeded by discovery. +package testutil + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/v10networks/nmon/internal/clock" + "github.com/v10networks/nmon/internal/discovery" + "github.com/v10networks/nmon/internal/secrets" + "github.com/v10networks/nmon/internal/simagent" + "github.com/v10networks/nmon/internal/snmp" + "github.com/v10networks/nmon/internal/store" +) + +// OpenStore returns a migrated SQLite store in a temp dir, closed at test end. +func OpenStore(t testing.TB, clk clock.Clock) store.Store { + t.Helper() + dir := t.TempDir() + key := filepath.Join(dir, "secret.key") + if err := secrets.GenerateKeyFile(key); err != nil { + t.Fatal(err) + } + kr, err := secrets.LoadKeyring(key) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + st, err := store.Open(ctx, store.DriverSQLite, filepath.Join(dir, "inventory.db"), kr, clk) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + if err := st.Migrate(ctx); err != nil { + t.Fatal(err) + } + return st +} + +// StartAgent serves cfg on [::1] (unless cfg.Addr says otherwise) until the test ends. +func StartAgent(t testing.TB, cfg simagent.Config) *simagent.Agent { + t.Helper() + a, err := simagent.Start(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.Close() }) + return a +} + +// AddDevice creates a v2c credential with the given community and a network +// device pointing at ::1, and seeds its interfaces from agent a. +func AddDevice(t testing.TB, st store.Store, a *simagent.Agent, name, community string, clk clock.Clock) *store.Device { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + cred := &store.SNMPCredential{Name: name + "-cred", Version: "v2c", Community: community} + if err := st.CreateCredential(ctx, cred); err != nil { + t.Fatal(err) + } + dev := &store.Device{Name: name, Kind: store.KindNetwork, SNMPCredentialID: &cred.ID, Enabled: true, Profile: "generic"} + if err := st.CreateDevice(ctx, dev); err != nil { + t.Fatal(err) + } + if err := st.SetAddresses(ctx, dev.ID, []string{"::1"}); err != nil { + t.Fatal(err) + } + Seed(t, st, a, dev, cred, clk) + got, err := st.GetDeviceByID(ctx, dev.ID) + if err != nil { + t.Fatal(err) + } + return got +} + +// Seed runs discovery.Seed for dev against agent a. +func Seed(t testing.TB, st store.Store, a *simagent.Agent, dev *store.Device, cred *store.SNMPCredential, clk clock.Clock) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + tg := discovery.Target(a.Addr().Addr(), int(a.Addr().Port()), cred, clk) + sess, err := snmp.Dial(ctx, tg, "discovery") + if err != nil { + t.Fatal(err) + } + defer sess.Close() + if err := discovery.Seed(ctx, st, sess, dev, clk); err != nil { + t.Fatal(err) + } +} diff --git a/scripts/gate-m1.sh b/scripts/gate-m1.sh new file mode 100755 index 0000000..bd51ba2 --- /dev/null +++ b/scripts/gate-m1.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Gate M1: `nmon serve` polls the counters fixture and /api/v1/query returns +# rising ifHCInOctets. Procedure: docs/milestones/M1-skeleton-and-fixtures.md. +# Needs: go, make, curl, file, ss, pgrep, python3 (JSON checks). +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +TMP="$(mktemp -d)" +FAILS=0 +NMON_PID="" AGENT_PID="" + +pass() { echo "PASS: $*"; } +fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); } +die() { fail "$*"; exit 1; } + +cleanup() { + local pid + for pid in $NMON_PID $AGENT_PID; do + kill "$pid" 2>/dev/null || true + done + for pid in $NMON_PID $AGENT_PID; do + for _ in $(seq 1 100); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done + kill -9 "$pid" 2>/dev/null || true + done + if [ "$FAILS" -ne 0 ] && [ -f "$TMP/nmon.log" ]; then + echo "--- tail of nmon.log"; tail -n 40 "$TMP/nmon.log" + fi + rm -rf "$TMP" +} +trap cleanup EXIT +trap 'exit 130' INT TERM + +pick_port() { + local p + while :; do + p=$((20000 + RANDOM % 30000)) + ss -H -lntu 2>/dev/null | awk '{print $5}' | grep -q "[:.]$p\$" || { echo "$p"; return; } + done +} + +command -v python3 >/dev/null || die "python3 is required for the JSON checks" + +# 1. Build with the embedded stores; the binary must be static. +echo "== build" +make build >"$TMP/build.log" 2>&1 || { tail -n 20 "$TMP/build.log"; die "make build"; } +if file nmon | grep -q 'statically linked'; then pass "nmon is statically linked"; else die "nmon is not statically linked: $(file nmon)"; fi + +# 2. Fixture agent on [::1]:. +echo "== simagent" +SNMP_PORT="$(pick_port)" +go build -o "$TMP/simagent" ./internal/simagent/cmd/simagent || die "build simagent" +"$TMP/simagent" -addr "[::1]:$SNMP_PORT" -rec testdata/snmprec/counters.snmprec >"$TMP/simagent.log" 2>&1 & +AGENT_PID=$! +for _ in $(seq 1 50); do grep -q listening "$TMP/simagent.log" 2>/dev/null && break; sleep 0.1; done +grep -q listening "$TMP/simagent.log" || { cat "$TMP/simagent.log"; die "simagent did not start"; } +pass "simagent serving counters.snmprec on [::1]:$SNMP_PORT" + +# 3. nmon serve, data dir and key file in separate places. +echo "== serve" +HTTP_PORT="$(pick_port)" +BASE="http://[::1]:$HTTP_PORT" +./nmon serve --data-dir "$TMP/data" --secret-key-file "$TMP/keys/secret.key" --insecure-http \ + --listen "[::1]:$HTTP_PORT" --poll-interval 10s --snmp-port "$SNMP_PORT" >"$TMP/nmon.log" 2>&1 & +NMON_PID=$! +for _ in $(seq 1 300); do + curl -fsS "$BASE/openapi.json" >/dev/null 2>&1 && break + kill -0 "$NMON_PID" 2>/dev/null || break + sleep 0.2 +done +curl -fsS "$BASE/openapi.json" >/dev/null 2>&1 || die "nmon serve did not come up" +pass "nmon serve is up on $BASE" + +# 4. Add the fixture device. +echo "== add device" +CODE="$(curl -sS -o "$TMP/device.json" -w '%{http_code}' -X POST "$BASE/api/v1/devices" \ + -H 'Content-Type: application/json' \ + -d '{"name":"counters","addresses":["::1"],"snmp":{"version":"v2c","community":"public"}}')" +[ "$CODE" = 201 ] || { cat "$TMP/device.json"; die "POST /api/v1/devices returned $CODE"; } +SLUG="$(python3 -I -c 'import json,sys; print(json.load(open(sys.argv[1]))["slug"])' "$TMP/device.json")" +NIF="$(python3 -I -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["interfaces"]))' "$TMP/device.json")" +[ "$NIF" -ge 1 ] || die "device $SLUG has no interfaces" +pass "device $SLUG added with $NIF interfaces" + +# 5. Three cycles, then query twice one cycle apart. +echo "== query (waiting for 3 poll cycles)" +sleep 30 +query() { curl -fsS --get "$BASE/api/v1/query" --data-urlencode 'q=ifHCInOctets' -o "$1"; } +for _ in $(seq 1 15); do + query "$TMP/q1.json" && python3 -I -c 'import json,sys; sys.exit(0 if json.load(open(sys.argv[1]))["data"]["result"] else 1)' "$TMP/q1.json" && break + sleep 2 +done +sleep 10 + +CHECK=' +import json, sys +a = json.load(open(sys.argv[1]))["data"]["result"] +b = {s["metric"].get("iface"): float(s["value"][1]) for s in json.load(open(sys.argv[2]))["data"]["result"]} +if not a: + print("no ifHCInOctets series"); sys.exit(1) +for s in a: + keys = sorted(s["metric"]) + if keys != ["__name__", "device", "iface"]: + print("labels are", keys); sys.exit(1) + i = s["metric"]["iface"] + if i not in b or not b[i] > float(s["value"][1]): + print("iface", i, "did not rise:", s["value"][1], "->", b.get(i)); sys.exit(1) +print(len(a), "series, labels exactly {__name__, device, iface}, all values strictly rising") +' +OK=0 +for _ in $(seq 1 6); do + query "$TMP/q2.json" && OUT="$(python3 -I -c "$CHECK" "$TMP/q1.json" "$TMP/q2.json" 2>&1)" && { OK=1; break; } + sleep 2 +done +if [ "$OK" = 1 ]; then pass "ifHCInOctets: $OUT"; else fail "ifHCInOctets check: ${OUT:-query failed}"; fi + +# 6. The metric stores listen on loopback only. +echo "== listeners" +PIDS="$(pgrep -P "$NMON_PID" | tr '\n' ' ')" +N=0 BAD=0 +while read -r addr pid; do + N=$((N + 1)) + case "$addr" in 127.0.0.1:*) ;; *) BAD=$((BAD + 1)); echo " store pid $pid listens on $addr" ;; esac +done < <(ss -H -ltnp | awk '{ + n = split($0, parts, "pid="); for (i = 2; i <= n; i++) { split(parts[i], p, ","); print $4, p[1] } }' | + while read -r addr pid; do for c in $PIDS; do [ "$c" = "$pid" ] && echo "$addr $pid"; done; done) +if [ "$N" -ge 2 ] && [ "$BAD" -eq 0 ]; then pass "$N store listeners, all on 127.0.0.1" +elif [ "$N" -lt 2 ]; then fail "found $N store listeners (children: ${PIDS:-none}), expected hot and long" +else fail "$BAD store listeners are not on 127.0.0.1"; fi + +# 7. Graceful shutdown. +echo "== shutdown" +kill -TERM "$NMON_PID" +if wait "$NMON_PID"; then pass "nmon exited cleanly on SIGTERM"; else fail "nmon exited with status $?"; fi +NMON_PID="" + +if [ "$FAILS" -eq 0 ]; then echo "GATE M1: PASS"; else echo "GATE M1: FAIL ($FAILS)"; exit 1; fi