M1 WP5: seed discovery, poller, API, serve wiring, gate script; M1 gate passed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
This commit is contained in:
2026-10-22 17:58:26 +00:00
parent 2f8710f7f3
commit 52fdd8d445
24 changed files with 3394 additions and 4 deletions

View File

@@ -43,7 +43,7 @@ test-pg:
test-v6only:
NMON_TEST_V6ONLY=1 $(GO) test -race -tags 'integration $(TAGS)' ./...
gate-m%:
gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6: gate-m%:
./scripts/gate-m$*.sh
clean:

60
cmd/nmon/migrate.go Normal file
View File

@@ -0,0 +1,60 @@
package main
import (
"context"
"fmt"
"os"
"strings"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/config"
"github.com/v10networks/nmon/internal/store"
)
// migrate applies pending inventory migrations (up, the default) or reverts the
// latest group (down). Credentials are not touched, so no key file is needed.
func migrate(ctx context.Context, args []string) error {
action := "up"
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
action, args = args[0], args[1:]
}
if action != "up" && action != "down" {
return fmt.Errorf("unknown action %q: want up or down", action)
}
var f config.Flags
fs := newFlagSet("migrate", "migrate [up|down] --data-dir DIR [flags]")
storeFlags(fs, &f)
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
if f.DataDir == "" {
return fmt.Errorf("--data-dir is required")
}
if err := os.MkdirAll(f.DataDir, 0o755); err != nil {
return err
}
dsn := f.DBDSN
if dsn == "" {
if f.DBDriver != store.DriverSQLite {
return fmt.Errorf("--db-dsn is required for --db-driver %s", f.DBDriver)
}
dsn = config.InventoryDB(f.DataDir)
}
st, err := store.Open(ctx, f.DBDriver, dsn, nil, clock.Real())
if err != nil {
return err
}
defer st.Close()
if action == "down" {
if err := st.Rollback(ctx); err != nil {
return err
}
fmt.Println("reverted the latest migration group")
return nil
}
if err := st.Migrate(ctx); err != nil {
return err
}
fmt.Println("migrations up to date")
return nil
}

77
cmd/nmon/serve.go Normal file
View File

@@ -0,0 +1,77 @@
package main
import (
"context"
"flag"
"fmt"
"log/slog"
"os"
"github.com/v10networks/nmon/internal/app"
"github.com/v10networks/nmon/internal/config"
)
func init() {
register("serve", "run the monitor: collect, store and serve the API", serve)
register("migrate", "apply (up) or revert (down) inventory migrations", migrate)
}
// hidden flags are dev/test only and left out of the usage text.
var hidden = map[string]bool{"poll-interval": true, "snmp-port": true}
// newFlagSet returns a FlagSet whose usage skips hidden flags.
func newFlagSet(name, usage string) *flag.FlagSet {
fs := flag.NewFlagSet(name, flag.ContinueOnError)
fs.Usage = func() {
fmt.Fprintf(fs.Output(), "usage: nmon %s\n\nflags:\n", usage)
fs.VisitAll(func(f *flag.Flag) {
if hidden[f.Name] {
return
}
def := ""
if f.DefValue != "" && f.DefValue != "false" && f.DefValue != "0" {
def = fmt.Sprintf(" (default %s)", f.DefValue)
}
fmt.Fprintf(fs.Output(), " --%s\n %s%s\n", f.Name, f.Usage, def)
})
}
return fs
}
// storeFlags registers the flags every inventory-using command shares.
func storeFlags(fs *flag.FlagSet, f *config.Flags) {
fs.StringVar(&f.DataDir, "data-dir", "", "data folder: inventory.db, vm-hot/, vm-long/, bin/, profiles/, config.yaml (required)")
fs.StringVar(&f.DBDriver, "db-driver", "sqlite", "inventory database: sqlite, postgres or mysql")
fs.StringVar(&f.DBDSN, "db-dsn", "", "database DSN; default <data-dir>/inventory.db for sqlite")
}
func serve(ctx context.Context, args []string) error {
var f config.Flags
fs := newFlagSet("serve", "serve --data-dir DIR --insecure-http [flags]")
storeFlags(fs, &f)
fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile, "SNMP credential key file; created on first run; must be outside --data-dir")
fs.StringVar(&f.Listen, "listen", app.DefaultListen, "address for the API")
fs.BoolVar(&f.InsecureHTTP, "insecure-http", false, "serve plain HTTP with no authentication (required until TLS and auth arrive in M6)")
fs.StringVar(&f.VMHotURL, "vm-hot-url", "", "use this VictoriaMetrics instance as the hot store instead of the embedded one")
fs.StringVar(&f.VMLongURL, "vm-long-url", "", "use this VictoriaMetrics instance as the long store instead of the embedded one")
fs.IntVar(&f.MaxInflight, "max-inflight", 512, "most SNMP polls in flight at once")
fs.DurationVar(&f.PollInterval, "poll-interval", 0, "dev: SNMP poll interval (default 60s)")
fs.IntVar(&f.SNMPPort, "snmp-port", 0, "dev: UDP port of every device's agent (default 161)")
if err := fs.Parse(args); err != nil {
return flagErr(err)
}
if f.DataDir == "" {
return fmt.Errorf("--data-dir is required")
}
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})))
slog.Info("nmon starting", "data_dir", f.DataDir, "listen", f.Listen)
return app.Run(ctx, f)
}
// flagErr turns flag.ErrHelp into a clean exit.
func flagErr(err error) error {
if err == flag.ErrHelp {
os.Exit(0)
}
return err
}

View File

@@ -90,4 +90,22 @@ WP2, WP3 and WP4 run in parallel. WP5 runs after them.
## Gate log
_(filled in by the supervisor when the gate passes)_
**Passed 2026-10-22**, run by the supervisor (Opus) on linux/amd64 (16 cores, 15 GB), Go 1.27.1, VictoriaMetrics v1.153.0.
| Check | Result |
|---|---|
| `gofmt -l`, `go vet ./...` | clean |
| `go test -race ./...` | 13 packages ok |
| `go test -race -tags 'integration vmembed' ./...` | all ok, including real-VM supervisor tests (kill -9 → healthy in about 1.4 s, listeners on 127.0.0.1 only) and in-process `app.Run` |
| `./scripts/test-pg.sh` (Postgres 16) | store suite ok on SQLite and Postgres; MySQL 8 suite also run once by WP2 |
| `make gate-m1` | `GATE M1: PASS`: static binary; counters fixture on `[::1]`; 8 `ifHCInOctets` series with labels exactly `{__name__, device, iface}`, all strictly rising; 2 store listeners, both on 127.0.0.1; clean SIGTERM exit |
**Decisions taken in M1** (beyond DEVIATIONS 1–5):
- `POST /api/v1/devices` walks the device before writing. An unreachable device returns 502 and nothing is stored, so no slug is burned.
- The 100k-series query cap is enforced on the result (422), not with a store flag, because rollups need more headroom.
- Supervisor: 3 restarts in 10 min are allowed; the 4th failure raises `store_failed` and stops the loop. The children run with `-search.latencyOffset=0s`.
- The init migration uses frozen struct copies. Credential ciphertext is bound to its row (AAD includes the row id).
- The dev-only flags `--poll-interval` and `--snmp-port` exist for gates and tests.
**Carried into M2:** a per-device SNMP port (a `devices.snmp_port` column replacing the global `--snmp-port`, which the loadtest needs); varbind adaptation, backoff and degraded state; the 30 s poll for 32-bit fast links; periodic re-discovery; full identity matching (`TODO(M2)` in `internal/discovery`).
**Carried into M3:** `/docs` currently loads Swagger UI assets from a CDN; self-host them with the embedded UI.

View File

@@ -11,8 +11,8 @@ its **Gate log** is filled in when the gate passes. A milestone is done only whe
| # | Milestone | Gate | Doc | Status |
|---|---|---|---|---|
| M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | in progress |
| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | planned |
| M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | **passed**, tagged `m1` |
| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | in progress (WP1 done) |
| M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | planned |
| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | planned |
| M5 | Push ingest, rollup, checks | a replayed Proxmox sample shows under its node and a 2 y query hits the 1 h tier | [M5](M5-ingest-rollup-checks.md) | planned |

1
go.mod
View File

@@ -3,6 +3,7 @@ module github.com/v10networks/nmon
go 1.27.1
require (
github.com/danielgtaylor/huma/v2 v2.39.1
github.com/go-sql-driver/mysql v1.10.1
github.com/gosnmp/gosnmp v1.46.0
github.com/oklog/ulid/v2 v2.1.2

8
go.sum
View File

@@ -2,8 +2,14 @@ filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/danielgtaylor/huma/v2 v2.39.1 h1:0kwF4ltQoYZ+IU55VPy+BcGekzgF44R64daTGde1H+g=
github.com/danielgtaylor/huma/v2 v2.39.1/go.mod h1:zcnQ38duIJ3VUHwFaBoZ6x8T+KN/mr33oyqxcj0HTug=
github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78=
github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -51,6 +57,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=

154
internal/api/api.go Normal file
View File

@@ -0,0 +1,154 @@
// Package api is nmon's HTTP API: huma v2 on net/http, errors as RFC 9457
// problem details, OpenAPI generated from the handlers at /openapi.json.
package api
import (
"context"
"encoding/json"
"errors"
"log/slog"
"net/http"
"strconv"
"strings"
"time"
"github.com/danielgtaylor/huma/v2"
"github.com/danielgtaylor/huma/v2/adapters/humago"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/selfmon"
"github.com/v10networks/nmon/internal/store"
)
// Limits.
const (
// QueryTimeout bounds one /query call.
QueryTimeout = 30 * time.Second
// MaxSeries is the most series a /query call may return; more is a 422.
// It is enforced on the result rather than with a store flag because the
// rollup job needs the store's own limit far higher (-search.maxUniqueTimeseries).
MaxSeries = 100_000
// DiscoveryTimeout bounds the seed discovery run by POST /devices.
DiscoveryTimeout = 30 * time.Second
)
// Querier runs read-only MetricsQL against the hot store. *vmsup.Querier implements it.
type Querier interface {
Query(ctx context.Context, store, q string, ts time.Time) ([]byte, error)
QueryRange(ctx context.Context, store, q string, from, to time.Time, step time.Duration) ([]byte, error)
}
// Refresher is told when the device list changed. *poller.Poller implements it.
type Refresher interface{ Refresh() }
// Deps are what the handlers need. Store, Querier and Poller are required.
type Deps struct {
Store store.Store
Querier Querier
Poller Refresher
Clock clock.Clock // default clock.Real()
Registry *selfmon.Registry // served at /metrics; default selfmon.Default
Logger *slog.Logger // default slog.Default()
SNMPPort int // dev/test override of UDP 161; 0 = 161
// DiscoveryTimeout overrides the 30 s seed discovery limit (tests).
DiscoveryTimeout time.Duration
}
type server struct {
Deps
}
// New returns the API handler with every route registered.
func New(d Deps) http.Handler {
if d.Clock == nil {
d.Clock = clock.Real()
}
if d.Registry == nil {
d.Registry = selfmon.Default
}
if d.Logger == nil {
d.Logger = slog.Default()
}
if d.DiscoveryTimeout <= 0 {
d.DiscoveryTimeout = DiscoveryTimeout
}
s := &server{d}
mux := http.NewServeMux()
cfg := huma.DefaultConfig("nmon", "1.0.0")
cfg.Info.Description = "nmon network monitor API."
cfg.OpenAPIPath = "/openapi"
cfg.DocsPath = "/docs"
cfg.DocsRenderer = huma.DocsRendererSwaggerUI
cfg.CreateHooks = nil // no $schema fields or Link headers in responses
api := humago.New(mux, cfg)
s.registerDevices(api)
s.registerQuery(api)
s.registerMetrics(api)
return problemFallback(mux)
}
// storeErr maps store sentinels to problem responses; anything else is a 500
// whose detail does not leak internals.
func (s *server) storeErr(err error) error {
switch {
case errors.Is(err, store.ErrNotFound):
return huma.Error404NotFound("not found")
case errors.Is(err, store.ErrConflict):
return huma.Error409Conflict(trimSentinel(err, store.ErrConflict))
case errors.Is(err, store.ErrInvalid):
return huma.Error422UnprocessableEntity(trimSentinel(err, store.ErrInvalid))
}
s.Logger.Error("store error", "err", err)
return huma.Error500InternalServerError("internal error")
}
// trimSentinel returns err's text without the leading "store: x: " sentinel.
func trimSentinel(err, sentinel error) string {
msg := err.Error()
if rest, ok := strings.CutPrefix(msg, sentinel.Error()+": "); ok {
return rest
}
return strings.TrimPrefix(msg, "store: ")
}
// problemFallback answers requests that match no route (404, 405) with
// problem+json instead of the mux's plain text.
func problemFallback(mux *http.ServeMux) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h, pattern := mux.Handler(r)
if pattern != "" {
mux.ServeHTTP(w, r) // sets the path values
return
}
cw := &capture{h: http.Header{}}
h.ServeHTTP(cw, r)
status := cw.status
if status == 0 {
status = http.StatusNotFound
}
if allow := cw.h.Get("Allow"); allow != "" {
w.Header().Set("Allow", allow)
}
w.Header().Set("Content-Type", "application/problem+json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{
"title": http.StatusText(status), "status": status,
"detail": r.Method + " " + r.URL.Path + ": " + strings.ToLower(http.StatusText(status)),
})
})
}
// capture records only the status and headers the mux would have written.
type capture struct {
h http.Header
status int
}
func (c *capture) Header() http.Header { return c.h }
func (c *capture) Write(b []byte) (int, error) { return len(b), nil }
func (c *capture) WriteHeader(s int) { c.status = s }
func itoa(n int) string { return strconv.Itoa(n) }

272
internal/api/api_test.go Normal file
View File

@@ -0,0 +1,272 @@
package api_test
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/v10networks/nmon/internal/api"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/fixturegen"
"github.com/v10networks/nmon/internal/selfmon"
"github.com/v10networks/nmon/internal/simagent"
"github.com/v10networks/nmon/internal/store"
"github.com/v10networks/nmon/internal/testutil"
"github.com/v10networks/nmon/internal/ulid"
"github.com/v10networks/nmon/internal/vmsup"
)
type querier struct {
body []byte
err error
q string
ts, from, to time.Time
step time.Duration
instant, rangeCnt int
}
func (q *querier) Query(_ context.Context, st, expr string, ts time.Time) ([]byte, error) {
q.instant++
q.q, q.ts = expr, ts
return q.body, q.err
}
func (q *querier) QueryRange(_ context.Context, st, expr string, from, to time.Time, step time.Duration) ([]byte, error) {
q.rangeCnt++
q.q, q.from, q.to, q.step = expr, from, to, step
return q.body, q.err
}
type refresher struct{ n atomic.Int32 }
func (r *refresher) Refresh() { r.n.Add(1) }
type env struct {
h http.Handler
st store.Store
q *querier
ref *refresher
reg *selfmon.Registry
}
func newEnv(t *testing.T, port int) *env {
clk := clock.NewFake(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC))
e := &env{st: testutil.OpenStore(t, clk), q: &querier{body: []byte(`{"status":"success","data":{"resultType":"vector","result":[]}}`)},
ref: &refresher{}, reg: selfmon.New()}
e.h = api.New(api.Deps{Store: e.st, Querier: e.q, Poller: e.ref, Clock: clk, Registry: e.reg, SNMPPort: port,
DiscoveryTimeout: 3 * time.Second})
return e
}
func (e *env) do(method, path, body string) *httptest.ResponseRecorder {
req := httptest.NewRequest(method, path, strings.NewReader(body))
if body != "" {
req.Header.Set("Content-Type", "application/json")
}
rec := httptest.NewRecorder()
e.h.ServeHTTP(rec, req)
return rec
}
func isProblem(t *testing.T, rec *httptest.ResponseRecorder, status int) {
t.Helper()
if rec.Code != status {
t.Fatalf("status %d, want %d: %s", rec.Code, status, rec.Body)
}
if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/problem+json") {
t.Fatalf("content type %q, want problem+json: %s", ct, rec.Body)
}
var p struct {
Title string `json:"title"`
Status int `json:"status"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &p); err != nil || p.Status != status || p.Title == "" {
t.Fatalf("problem body %s (%v)", rec.Body, err)
}
}
const community = "s3cr3t-comm-xyz"
func TestAddAndGetDevice(t *testing.T) {
a := testutil.StartAgent(t, simagent.Config{
Recording: fixturegen.Generic(fixturegen.Options{Ports: 8, SysName: "edge1", HCCounters: true}), Community: community,
})
e := newEnv(t, int(a.Addr().Port()))
body := fmt.Sprintf(`{"name":"Edge One","addresses":["::1"],"site":"yyz","role":"edge","snmp":{"version":"v2c","community":%q}}`, community)
rec := e.do("POST", "/api/v1/devices", body)
if rec.Code != http.StatusCreated {
t.Fatalf("POST: %d %s", rec.Code, rec.Body)
}
if e.ref.n.Load() != 1 {
t.Error("poller not refreshed")
}
var created api.DeviceView
if err := json.Unmarshal(rec.Body.Bytes(), &created); err != nil {
t.Fatal(err)
}
if created.Slug == "" || created.Name != "Edge One" || created.Site != "yyz" || created.SysName != "edge1" ||
created.MgmtIP != "::1" || created.SNMPVersion != "v2c" || len(created.Interfaces) != 8 {
t.Fatalf("created: %+v", created)
}
get := e.do("GET", "/api/v1/devices/"+created.Slug, "")
if get.Code != 200 {
t.Fatalf("GET: %d %s", get.Code, get.Body)
}
for _, r := range []*httptest.ResponseRecorder{rec, get} {
if strings.Contains(r.Body.String(), community) {
t.Fatalf("response contains the community: %s", r.Body)
}
var m map[string]json.RawMessage
if err := json.Unmarshal(r.Body.Bytes(), &m); err != nil {
t.Fatal(err)
}
for _, k := range []string{"id", "snmp_credential_id", "community", "device_id"} {
if _, ok := m[k]; ok {
t.Errorf("device response has key %q", k)
}
}
}
var got api.DeviceView
if err := json.Unmarshal(get.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
for _, i := range got.Interfaces {
if !ulid.Valid(i.ID) {
t.Errorf("interface id %q", i.ID)
}
if i.FirstSeen.Location() != time.UTC {
t.Errorf("timestamp not UTC: %v", i.FirstSeen)
}
}
if !strings.Contains(get.Body.String(), `"created_at":"2026-10-22T12:00:00Z"`) {
t.Errorf("created_at is not RFC 3339 UTC: %s", get.Body)
}
// the secret is in the store, encrypted, and decrypts for the poller
d, _ := e.st.GetDeviceBySlug(context.Background(), created.Slug)
c, err := e.st.GetCredential(context.Background(), *d.SNMPCredentialID)
if err != nil || c.Community != community {
t.Fatalf("stored credential: %+v %v", c, err)
}
}
func TestAddDeviceRejections(t *testing.T) {
e := newEnv(t, 1) // nothing listens on UDP 1
for name, tc := range map[string]struct {
body string
status int
}{
"no snmp for network": {`{"name":"x","addresses":["::1"]}`, 422},
"bad address": {`{"name":"x","addresses":["nope"],"snmp":{"version":"v2c","community":"c"}}`, 422},
"no community": {`{"name":"x","addresses":["::1"],"snmp":{"version":"v2c"}}`, 422},
"bad version": {`{"name":"x","addresses":["::1"],"snmp":{"version":"v9","community":"c"}}`, 422},
"no addresses": {`{"name":"x","addresses":[],"snmp":{"version":"v2c","community":"c"}}`, 422},
"unreachable": {`{"name":"x","addresses":["::1"],"snmp":{"version":"v2c","community":"c"}}`, 502},
} {
t.Run(name, func(t *testing.T) { isProblem(t, e.do("POST", "/api/v1/devices", tc.body), tc.status) })
}
if _, n, err := e.st.ListDevices(context.Background(), store.DeviceFilter{}, store.Page{}); err != nil || n != 0 {
t.Fatalf("rejected requests stored %d devices (%v)", n, err)
}
if e.ref.n.Load() != 0 {
t.Error("poller refreshed for a rejected device")
}
// a device without SNMP is fine
rec := e.do("POST", "/api/v1/devices", `{"name":"vm1","kind":"vm","addresses":["192.0.2.7"]}`)
if rec.Code != 201 {
t.Fatalf("vm: %d %s", rec.Code, rec.Body)
}
}
func TestProblemDetails(t *testing.T) {
e := newEnv(t, 0)
isProblem(t, e.do("GET", "/api/v1/devices/no-such-device", ""), 404)
isProblem(t, e.do("GET", "/api/v1/nothing", ""), 404)
rec := e.do("DELETE", "/api/v1/query", "")
isProblem(t, rec, 405)
isProblem(t, e.do("GET", "/api/v1/query", ""), 422) // missing q
}
func TestOpenAPIDocsAndMetrics(t *testing.T) {
e := newEnv(t, 0)
e.reg.Counter("nmon_test_total", "device", "d1").Add(3)
rec := e.do("GET", "/openapi.json", "")
var spec struct {
Paths map[string]map[string]any `json:"paths"`
}
if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &spec) != nil {
t.Fatalf("openapi.json: %d %s", rec.Code, rec.Body)
}
for _, p := range []string{"/api/v1/devices", "/api/v1/devices/{slug}", "/api/v1/query", "/api/v1/query_range", "/metrics"} {
if spec.Paths[p] == nil {
t.Errorf("openapi.json lacks %s", p)
}
}
if d := e.do("GET", "/docs", ""); d.Code != 200 || !strings.Contains(d.Header().Get("Content-Type"), "text/html") {
t.Errorf("/docs: %d %s", d.Code, d.Header().Get("Content-Type"))
}
m := e.do("GET", "/metrics", "")
if m.Code != 200 || !strings.Contains(m.Body.String(), `nmon_test_total{device="d1"} 3`) {
t.Errorf("/metrics: %d %s", m.Code, m.Body)
}
}
func seriesBody(n int) []byte {
var b bytes.Buffer
b.WriteString(`{"status":"success","data":{"resultType":"vector","result":[`)
for i := 0; i < n; i++ {
if i > 0 {
b.WriteByte(',')
}
b.WriteString(`{"metric":{},"value":[1,"1"]}`)
}
b.WriteString(`]}}`)
return b.Bytes()
}
func TestQueryProxy(t *testing.T) {
e := newEnv(t, 0)
e.q.body = seriesBody(3)
rec := e.do("GET", "/api/v1/query?q=ifHCInOctets&time=2026-10-22T12:00:00Z", "")
if rec.Code != 200 || !bytes.Equal(rec.Body.Bytes(), e.q.body) || rec.Header().Get("Content-Type") != "application/json" {
t.Fatalf("passthrough: %d %s", rec.Code, rec.Body)
}
if e.q.q != "ifHCInOctets" || !e.q.ts.Equal(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)) {
t.Errorf("forwarded %q at %v", e.q.q, e.q.ts)
}
e.do("GET", "/api/v1/query?q=up&time=1700000000.5", "")
if e.q.ts.UnixMilli() != 1700000000500 {
t.Errorf("unix time parsed as %v", e.q.ts)
}
isProblem(t, e.do("GET", "/api/v1/query?q=up&time=yesterday", ""), 422)
rec = e.do("GET", "/api/v1/query_range?q=up&from=1700000000&to=1700003600&step=30s", "")
if rec.Code != 200 || e.q.step != 30*time.Second || e.q.to.Sub(e.q.from) != time.Hour {
t.Fatalf("range: %d %s step=%v", rec.Code, rec.Body, e.q.step)
}
isProblem(t, e.do("GET", "/api/v1/query_range?q=up", ""), 422) // step required
isProblem(t, e.do("GET", "/api/v1/query_range?q=up&step=0", ""), 422)
e.q.body = seriesBody(api.MaxSeries)
if rec := e.do("GET", "/api/v1/query?q=x", ""); rec.Code != 200 {
t.Fatalf("exactly %d series must pass: %d", api.MaxSeries, rec.Code)
}
e.q.body = seriesBody(api.MaxSeries + 1)
isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 422)
isProblem(t, e.do("GET", "/api/v1/query_range?q=x&step=1m", ""), 422)
e.q.err = fmt.Errorf("%w: hot", vmsup.ErrStoreDown)
isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 503)
e.q.err = errors.New("vmsup: hot store: 422 Unprocessable Entity: cannot parse")
isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 422)
e.q.err = errors.New("connection reset")
isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 502)
}

281
internal/api/devices.go Normal file
View File

@@ -0,0 +1,281 @@
package api
import (
"context"
"fmt"
"net/netip"
"time"
"github.com/danielgtaylor/huma/v2"
"github.com/v10networks/nmon/internal/discovery"
"github.com/v10networks/nmon/internal/snmp"
"github.com/v10networks/nmon/internal/store"
"github.com/v10networks/nmon/internal/ulid"
)
// SNMPBody is the SNMP credential in a request. Secrets are write-only: no
// response ever contains them.
type SNMPBody struct {
Version string `json:"version" enum:"v1,v2c,v3" doc:"SNMP version."`
Community string `json:"community,omitempty" doc:"v1 and v2c community." example:"public"`
V3User string `json:"v3_user,omitempty" doc:"v3 user name."`
V3AuthProto string `json:"v3_auth_proto,omitempty" doc:"MD5, SHA, SHA224, SHA256 (default), SHA384 or SHA512."`
V3AuthPass string `json:"v3_auth_pass,omitempty"`
V3PrivProto string `json:"v3_priv_proto,omitempty" doc:"DES, AES (default), AES192 or AES256."`
V3PrivPass string `json:"v3_priv_pass,omitempty"`
V3Context string `json:"v3_context,omitempty"`
}
// AddDeviceBody is the body of POST /api/v1/devices.
type AddDeviceBody struct {
Name string `json:"name" minLength:"1" maxLength:"255" example:"core-sw1"`
Addresses []string `json:"addresses" minItems:"1" doc:"Management addresses; an IPv6 address is preferred over IPv4." example:"[\"2001:db8::1\"]"`
Kind string `json:"kind,omitempty" enum:"network,vm,container,check_only" default:"network"`
Site string `json:"site,omitempty"`
Role string `json:"role,omitempty"`
SNMP *SNMPBody `json:"snmp,omitempty" doc:"Required for kind network."`
}
// InterfaceView is an interface as the API shows it, addressed by ULID.
type InterfaceView struct {
ID string `json:"id" doc:"Immutable ULID."`
IfIndex int64 `json:"ifindex" doc:"Current ifIndex; may change, the ID does not."`
IfName string `json:"ifname"`
IfDescr string `json:"ifdescr"`
IfAlias string `json:"ifalias"`
IfPhysAddr string `json:"ifphysaddr"`
IfSpeed int64 `json:"ifspeed" doc:"bits per second"`
IfHighSpeed int64 `json:"ifhighspeed" doc:"megabits per second"`
IfType int64 `json:"iftype"`
AdminStatus int64 `json:"admin_status"`
OperStatus int64 `json:"oper_status"`
HasHCCounters bool `json:"has_hc_counters"`
Excluded bool `json:"excluded"`
FirstSeen time.Time `json:"first_seen"`
LastDiscovered time.Time `json:"last_discovered"`
}
// DeviceView is a device as the API shows it. It carries no numeric ID and no credential.
type DeviceView struct {
Slug string `json:"slug" doc:"Immutable public key."`
Name string `json:"name"`
Kind string `json:"kind"`
Site string `json:"site"`
Role string `json:"role"`
Profile string `json:"profile"`
Enabled bool `json:"enabled"`
MgmtIP string `json:"mgmt_ip"`
Addresses []string `json:"addresses"`
SNMPVersion string `json:"snmp_version,omitempty"`
SysName string `json:"sys_name"`
SysObjectID string `json:"sys_object_id"`
SysDescr string `json:"sys_descr"`
PollState string `json:"poll_state"`
Warnings []string `json:"warnings"`
LastSeen *time.Time `json:"last_seen,omitempty"`
CreatedAt time.Time `json:"created_at"`
RetiredAt *time.Time `json:"retired_at,omitempty"`
Interfaces []InterfaceView `json:"interfaces"`
}
type deviceOut struct{ Body DeviceView }
func (s *server) registerDevices(api huma.API) {
huma.Register(api, huma.Operation{
OperationID: "add-device", Method: "POST", Path: "/api/v1/devices", Tags: []string{"devices"},
Summary: "Add a device",
Description: "Creates the credential, device and addresses, runs seed discovery synchronously (30 s limit) and starts polling. A device that does not answer SNMP is rejected with 502 and nothing is stored.",
DefaultStatus: 201,
Errors: []int{409, 422, 502},
}, s.addDevice)
huma.Register(api, huma.Operation{
OperationID: "get-device", Method: "GET", Path: "/api/v1/devices/{slug}", Tags: []string{"devices"},
Summary: "Get a device and its interfaces",
Errors: []int{404},
}, s.getDevice)
}
func (s *server) addDevice(ctx context.Context, in *struct{ Body AddDeviceBody }) (*deviceOut, error) {
b := in.Body
kind := b.Kind
if kind == "" {
kind = store.KindNetwork
}
var addrs []netip.Addr
for _, a := range b.Addresses {
ip, err := netip.ParseAddr(a)
if err != nil {
return nil, huma.Error422UnprocessableEntity(fmt.Sprintf("address %q is not an IP address", a))
}
addrs = append(addrs, ip.WithZone("").Unmap())
}
if len(addrs) == 0 {
return nil, huma.Error422UnprocessableEntity("at least one address is required")
}
if kind == store.KindNetwork && b.SNMP == nil {
return nil, huma.Error422UnprocessableEntity("snmp credentials are required for kind network")
}
var (
cred *store.SNMPCredential
res *discovery.Result
)
if b.SNMP != nil {
var err error
if cred, err = s.credential(b.SNMP); err != nil {
return nil, err
}
pref := addrs[0]
for _, a := range addrs {
if a.Is6() {
pref = a
break
}
}
if res, err = s.collect(ctx, pref, cred); err != nil {
return nil, err
}
}
var slug string
err := s.Store.WithTx(ctx, func(tx store.Store) error {
d := &store.Device{Name: b.Name, Kind: kind, Site: b.Site, Role: b.Role, Enabled: true, Profile: "generic"}
if cred != nil {
if err := tx.CreateCredential(ctx, cred); err != nil {
return err
}
d.SNMPCredentialID = &cred.ID
}
if err := tx.CreateDevice(ctx, d); err != nil {
return err
}
strs := make([]string, len(addrs))
for i, a := range addrs {
strs[i] = a.String()
}
if err := tx.SetAddresses(ctx, d.ID, strs); err != nil {
return err
}
if res != nil {
if err := discovery.Apply(ctx, tx, d, res, s.Clock); err != nil {
return err
}
}
slug = d.Slug
return tx.AppendAudit(ctx, "anonymous", "device.add", d.Slug, nil,
map[string]any{"name": d.Name, "kind": d.Kind, "addresses": strs})
})
if err != nil {
return nil, s.storeErr(err)
}
s.Poller.Refresh()
return s.view(ctx, slug)
}
// credential validates the request's SNMP block and returns a fresh credential.
func (s *server) credential(b *SNMPBody) (*store.SNMPCredential, error) {
c := &store.SNMPCredential{
Name: "cred-" + ulid.New(s.Clock.Now()), Version: b.Version, Community: b.Community, V3User: b.V3User,
V3AuthProto: b.V3AuthProto, V3AuthPass: b.V3AuthPass, V3PrivProto: b.V3PrivProto, V3PrivPass: b.V3PrivPass,
V3Context: b.V3Context,
}
switch b.Version {
case "v1", "v2c":
if b.Community == "" {
return nil, huma.Error422UnprocessableEntity("snmp.community is required for " + b.Version)
}
case "v3":
if b.V3User == "" {
return nil, huma.Error422UnprocessableEntity("snmp.v3_user is required for v3")
}
if _, err := snmp.ParseAuthProto(b.V3AuthProto); err != nil {
return nil, huma.Error422UnprocessableEntity(err.Error())
}
if _, err := snmp.ParsePrivProto(b.V3PrivProto); err != nil {
return nil, huma.Error422UnprocessableEntity(err.Error())
}
default:
return nil, huma.Error422UnprocessableEntity("snmp.version must be v1, v2c or v3")
}
return c, nil
}
// collect runs the seed walk against addr before anything is stored.
func (s *server) collect(ctx context.Context, addr netip.Addr, cred *store.SNMPCredential) (*discovery.Result, error) {
dctx, cancel := context.WithTimeout(ctx, s.DiscoveryTimeout)
defer cancel()
sess, err := snmp.Dial(dctx, discovery.Target(addr, s.SNMPPort, cred, s.Clock), "discovery")
if err != nil {
return nil, huma.Error422UnprocessableEntity("snmp: " + err.Error())
}
defer sess.Close()
res, err := discovery.Collect(dctx, sess, s.Clock)
if err != nil {
return nil, huma.Error502BadGateway("discovery of " + addr.String() + " failed: " + err.Error())
}
return res, nil
}
func (s *server) getDevice(ctx context.Context, in *struct {
Slug string `path:"slug" doc:"Device slug."`
}) (*deviceOut, error) {
return s.view(ctx, in.Slug)
}
func (s *server) view(ctx context.Context, slug string) (*deviceOut, error) {
d, err := s.Store.GetDeviceBySlug(ctx, slug)
if err != nil {
return nil, s.storeErr(err)
}
if d.PurgedAt != nil {
return nil, huma.Error404NotFound("not found")
}
v := DeviceView{
Slug: d.Slug, Name: d.Name, Kind: d.Kind, Site: d.Site, Role: d.Role, Profile: d.Profile, Enabled: d.Enabled,
MgmtIP: d.MgmtIP, SysName: d.SysName, SysObjectID: d.SysObjectID, SysDescr: d.SysDescr,
PollState: d.PollState, Warnings: nonNil(d.Warnings), LastSeen: utcp(d.LastSeen), CreatedAt: d.CreatedAt.UTC(),
RetiredAt: utcp(d.RetiredAt), Addresses: []string{}, Interfaces: []InterfaceView{},
}
addrs, err := s.Store.ListAddresses(ctx, d.ID)
if err != nil {
return nil, s.storeErr(err)
}
for _, a := range addrs {
v.Addresses = append(v.Addresses, a.Addr)
}
if d.SNMPCredentialID != nil {
c, err := s.Store.GetCredential(ctx, *d.SNMPCredentialID)
if err != nil {
return nil, s.storeErr(err)
}
v.SNMPVersion = c.Version // the secrets stay behind
}
ifs, err := s.Store.ListInterfaces(ctx, d.ID, false)
if err != nil {
return nil, s.storeErr(err)
}
for _, i := range ifs {
v.Interfaces = append(v.Interfaces, InterfaceView{
ID: i.ID, IfIndex: i.IfIndex, IfName: i.IfName, IfDescr: i.IfDescr, IfAlias: i.IfAlias,
IfPhysAddr: i.IfPhysAddr, IfSpeed: i.IfSpeed, IfHighSpeed: i.IfHighSpeed, IfType: i.IfType,
AdminStatus: i.AdminStatus, OperStatus: i.OperStatus, HasHCCounters: i.HasHCCounters, Excluded: i.Excluded,
FirstSeen: i.FirstSeen.UTC(), LastDiscovered: i.LastDiscovered.UTC(),
})
}
return &deviceOut{Body: v}, nil
}
func nonNil(s []string) []string {
if s == nil {
return []string{}
}
return s
}
func utcp(t *time.Time) *time.Time {
if t == nil {
return nil
}
u := t.UTC()
return &u
}

View File

@@ -0,0 +1,29 @@
package api
import (
"fmt"
"testing"
"github.com/danielgtaylor/huma/v2"
"github.com/v10networks/nmon/internal/store"
)
func TestStoreErrorMapping(t *testing.T) {
s := &server{}
s.Logger = nil
for _, tc := range []struct {
err error
status int
detail string
}{
{fmt.Errorf("%w: slug taken", store.ErrConflict), 409, "slug taken"},
{fmt.Errorf("%w: bad kind", store.ErrInvalid), 422, "bad kind"},
{fmt.Errorf("wrapped: %w", store.ErrNotFound), 404, "not found"},
} {
se, ok := s.storeErr(tc.err).(huma.StatusError)
if !ok || se.GetStatus() != tc.status || se.Error() != tc.detail {
t.Errorf("%v: got %v, want %d %q", tc.err, se, tc.status, tc.detail)
}
}
}

169
internal/api/query.go Normal file
View File

@@ -0,0 +1,169 @@
package api
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"strconv"
"strings"
"time"
"github.com/danielgtaylor/huma/v2"
"github.com/v10networks/nmon/internal/vmsup"
)
var jsonObject = map[string]*huma.Response{
"200": {Description: "The store's Prometheus-compatible JSON response, unchanged.", Content: map[string]*huma.MediaType{
"application/json": {Schema: &huma.Schema{Type: "object"}},
}},
}
func (s *server) registerQuery(api huma.API) {
huma.Register(api, huma.Operation{
OperationID: "query", Method: "GET", Path: "/api/v1/query", Tags: []string{"query"},
Summary: "Instant MetricsQL query against the hot store",
Description: "Read-only proxy with a 30 s timeout. A result of more than 100,000 series is rejected with 422.",
Responses: jsonObject, Errors: []int{422, 503, 504},
}, s.query)
huma.Register(api, huma.Operation{
OperationID: "query-range", Method: "GET", Path: "/api/v1/query_range", Tags: []string{"query"},
Summary: "Range MetricsQL query against the hot store",
Description: "Same limits as /api/v1/query.",
Responses: jsonObject, Errors: []int{422, 503, 504},
}, s.queryRange)
}
func raw(body []byte) *huma.StreamResponse {
return &huma.StreamResponse{Body: func(ctx huma.Context) {
ctx.SetHeader("Content-Type", "application/json")
ctx.SetStatus(http.StatusOK)
_, _ = ctx.BodyWriter().Write(body)
}}
}
func (s *server) query(ctx context.Context, in *struct {
Q string `query:"q" required:"true" minLength:"1" doc:"MetricsQL expression." example:"ifHCInOctets"`
Time string `query:"time" doc:"Evaluation time: RFC 3339 or Unix seconds. Default now."`
}) (*huma.StreamResponse, error) {
var ts time.Time
if in.Time != "" {
var err error
if ts, err = parseTime(in.Time); err != nil {
return nil, huma.Error422UnprocessableEntity("time: " + err.Error())
}
}
qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
defer cancel()
body, err := s.Querier.Query(qctx, "hot", in.Q, ts)
return s.finish(qctx, body, err)
}
func (s *server) queryRange(ctx context.Context, in *struct {
Q string `query:"q" required:"true" minLength:"1" doc:"MetricsQL expression."`
From string `query:"from" doc:"Start: RFC 3339 or Unix seconds. Default one hour before to."`
To string `query:"to" doc:"End: RFC 3339 or Unix seconds. Default now."`
Step string `query:"step" required:"true" doc:"Resolution step: a duration such as 30s or a number of seconds."`
}) (*huma.StreamResponse, error) {
to := s.Clock.Now()
if in.To != "" {
var err error
if to, err = parseTime(in.To); err != nil {
return nil, huma.Error422UnprocessableEntity("to: " + err.Error())
}
}
from := to.Add(-time.Hour)
if in.From != "" {
var err error
if from, err = parseTime(in.From); err != nil {
return nil, huma.Error422UnprocessableEntity("from: " + err.Error())
}
}
step, err := parseStep(in.Step)
if err != nil {
return nil, huma.Error422UnprocessableEntity("step: " + err.Error())
}
if !from.Before(to) {
return nil, huma.Error422UnprocessableEntity("from must be before to")
}
qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
defer cancel()
body, err := s.Querier.QueryRange(qctx, "hot", in.Q, from, to, step)
return s.finish(qctx, body, err)
}
// finish applies the series cap and maps store errors to problems.
func (s *server) finish(ctx context.Context, body []byte, err error) (*huma.StreamResponse, error) {
switch {
case err == nil:
case errors.Is(err, vmsup.ErrStoreDown):
return nil, huma.Error503ServiceUnavailable("the metric store is not available")
case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded):
return nil, huma.Error504GatewayTimeout("query exceeded " + QueryTimeout.String())
case strings.Contains(err.Error(), "400 Bad Request") || strings.Contains(err.Error(), "422 Unprocessable"):
return nil, huma.Error422UnprocessableEntity(err.Error()) // the store rejected the query
default:
s.Logger.Warn("query failed", "err", err)
return nil, huma.Error502BadGateway("metric store error")
}
if n := countSeries(body); n > MaxSeries {
return nil, huma.Error422UnprocessableEntity("query returned " + itoa(n) + " series; the limit is " + itoa(MaxSeries) + ", narrow the selector")
}
return raw(body), nil
}
// countSeries returns len(data.result) of a Prometheus-style response.
func countSeries(body []byte) int {
var r struct {
Data struct {
Result []json.RawMessage `json:"result"`
} `json:"data"`
}
if json.NewDecoder(bytes.NewReader(body)).Decode(&r) != nil {
return 0
}
return len(r.Data.Result)
}
func parseTime(s string) (time.Time, error) {
if t, err := time.Parse(time.RFC3339, s); err == nil {
return t.UTC(), nil
}
f, err := strconv.ParseFloat(s, 64)
if err != nil {
return time.Time{}, errors.New("want RFC 3339 or Unix seconds")
}
return time.UnixMilli(int64(f * 1000)).UTC(), nil
}
func parseStep(s string) (time.Duration, error) {
d, err := time.ParseDuration(s)
if err != nil {
f, ferr := strconv.ParseFloat(s, 64)
if ferr != nil {
return 0, errors.New("want a duration such as 30s or a number of seconds")
}
d = time.Duration(f * float64(time.Second))
}
if d <= 0 {
return 0, errors.New("must be positive")
}
return d, nil
}
func (s *server) registerMetrics(api huma.API) {
huma.Register(api, huma.Operation{
OperationID: "metrics", Method: "GET", Path: "/metrics", Tags: []string{"internals"},
Summary: "nmon's own metrics in Prometheus text format",
Responses: map[string]*huma.Response{"200": {Description: "Prometheus text exposition.", Content: map[string]*huma.MediaType{
"text/plain": {Schema: &huma.Schema{Type: "string"}},
}}},
}, func(context.Context, *struct{}) (*huma.StreamResponse, error) {
return &huma.StreamResponse{Body: func(ctx huma.Context) {
ctx.SetHeader("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
_ = s.Registry.WritePrometheus(ctx.BodyWriter())
}}, nil
})
}

263
internal/app/app.go Normal file
View File

@@ -0,0 +1,263 @@
// Package app wires nmon serve: config, secret key, inventory, metric stores,
// write queues, poller and HTTP API, and shuts them down in order.
package app
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/v10networks/nmon/internal/api"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/config"
"github.com/v10networks/nmon/internal/poller"
"github.com/v10networks/nmon/internal/secrets"
"github.com/v10networks/nmon/internal/store"
"github.com/v10networks/nmon/internal/vmsup"
)
// Defaults for flags left empty.
const (
DefaultListen = "[::]:8443"
DefaultSecretKeyFile = "/etc/nmon/secret.key"
)
const (
startTimeout = 60 * time.Second
shutdownTimeout = 30 * time.Second
)
// App is a running nmon server.
type App struct {
log *slog.Logger
st store.Store
sup *vmsup.Supervisor
hot *vmsup.Queue
long *vmsup.Queue
poller *poller.Poller
srv *http.Server
ln net.Listener
bg context.CancelFunc // poller and queue goroutines
pollDone chan struct{}
queuesWG chan struct{}
serveErr chan error
}
// Run starts nmon and blocks until ctx is cancelled or the HTTP server fails,
// then shuts everything down gracefully.
func Run(ctx context.Context, f config.Flags) error {
a, err := Start(ctx, f)
if err != nil {
return err
}
var runErr error
select {
case <-ctx.Done():
case runErr = <-a.serveErr:
}
sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
return errors.Join(runErr, a.Stop(sctx))
}
// Start brings every component up and returns once the API is listening. ctx
// bounds startup only; call Stop to shut down.
func Start(ctx context.Context, f config.Flags) (_ *App, err error) {
log := slog.Default()
if !f.InsecureHTTP {
return nil, errors.New("TLS and authentication arrive in M6; until then pass --insecure-http to run without them (development only)")
}
if f.DataDir == "" {
return nil, errors.New("--data-dir is required")
}
if f.Listen == "" {
f.Listen = DefaultListen
}
if f.SecretKeyFile == "" {
f.SecretKeyFile = DefaultSecretKeyFile
}
if f.MaxInflight <= 0 {
f.MaxInflight = poller.DefaultMaxInflight
}
dataDir, err := filepath.Abs(f.DataDir)
if err != nil {
return nil, err
}
f.DataDir = dataDir
keyFile, err := filepath.Abs(f.SecretKeyFile)
if err != nil {
return nil, err
}
if inside(dataDir, keyFile) {
return nil, fmt.Errorf("--secret-key-file %s is inside --data-dir %s; a backup of the data dir must not contain the key", keyFile, dataDir)
}
if err := os.MkdirAll(dataDir, 0o755); err != nil {
return nil, err
}
cfg, err := config.Load(dataDir)
if err != nil {
return nil, err
}
driver := f.DBDriver
if driver == "" {
driver = store.DriverSQLite
}
dsn := f.DBDSN
if dsn == "" {
if driver != store.DriverSQLite {
return nil, fmt.Errorf("--db-dsn is required for --db-driver %s", driver)
}
dsn = config.InventoryDB(dataDir)
}
if err := ensureKey(log, keyFile, driver == store.DriverSQLite && fileExists(config.InventoryDB(dataDir))); err != nil {
return nil, err
}
kr, err := secrets.LoadKeyring(keyFile)
if err != nil {
return nil, fmt.Errorf("secret key: %w", err)
}
a := &App{log: log, serveErr: make(chan error, 1), pollDone: make(chan struct{}), queuesWG: make(chan struct{})}
defer func() {
if err != nil {
sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
_ = a.Stop(sctx)
}
}()
if a.st, err = store.Open(ctx, driver, dsn, kr, clock.Real()); err != nil {
return nil, fmt.Errorf("open inventory: %w", err)
}
if err = a.st.Migrate(ctx); err != nil {
return nil, fmt.Errorf("migrate inventory: %w", err)
}
a.sup, err = vmsup.New(vmsup.Options{
DataDir: dataDir, HotURL: f.VMHotURL, LongURL: f.VMLongURL,
HotRetention: cfg.Retention.Hot.D(), LongRetention: cfg.Retention.Long.D(), Logger: log,
OnEvent: func(e vmsup.Event) {
log.Warn("metric store event", "type", e.Type, "store", e.Store, "detail", e.Detail, "restarts", e.Restarts)
},
})
if err != nil {
return nil, err
}
sctx, cancel := context.WithTimeout(ctx, startTimeout)
defer cancel()
if err = a.sup.Start(sctx); err != nil {
a.sup = nil // Start leaves nothing running when it fails
return nil, fmt.Errorf("start metric stores: %w", err)
}
log.Info("metric stores up", "hot", a.sup.HotURL(), "long", a.sup.LongURL())
bg, cancelBG := context.WithCancel(context.Background())
a.bg = cancelBG
a.hot = vmsup.NewQueue(vmsup.StoreHot, a.sup, vmsup.QueueOptions{})
a.long = vmsup.NewQueue(vmsup.StoreLong, a.sup, vmsup.QueueOptions{})
qdone := make(chan struct{}, 2)
for _, q := range []*vmsup.Queue{a.hot, a.long} {
go func() { q.Run(bg); qdone <- struct{}{} }()
}
go func() { <-qdone; <-qdone; close(a.queuesWG) }()
a.poller = poller.New(poller.Options{
Store: a.st, Writer: a.hot, Interval: f.PollInterval, MaxInflight: f.MaxInflight, SNMPPort: f.SNMPPort, Logger: log,
})
go func() { _ = a.poller.Run(bg); close(a.pollDone) }()
handler := api.New(api.Deps{
Store: a.st, Querier: vmsup.NewQuerier(a.sup), Poller: a.poller, SNMPPort: f.SNMPPort, Logger: log,
})
if a.ln, err = net.Listen("tcp", f.Listen); err != nil {
return nil, fmt.Errorf("listen: %w", err)
}
a.srv = &http.Server{Handler: handler, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
go func() {
if err := a.srv.Serve(a.ln); !errors.Is(err, http.ErrServerClosed) {
a.serveErr <- err
}
}()
log.Warn("serving plain HTTP without authentication (--insecure-http); do not expose this port", "addr", a.ln.Addr().String())
return a, nil
}
// Addr is the address the API listens on.
func (a *App) Addr() string { return a.ln.Addr().String() }
// HotURL is the base URL of the hot metric store.
func (a *App) HotURL() string { return a.sup.HotURL() }
// Stop shuts down: HTTP first, then the poller, then the metric stores (which
// flushes the write queues), then the inventory. It is safe after a failed Start.
func (a *App) Stop(ctx context.Context) error {
var errs []error
if a.srv != nil {
errs = append(errs, a.srv.Shutdown(ctx))
}
if a.bg != nil {
a.bg()
<-a.pollDone
<-a.queuesWG
}
if a.sup != nil {
errs = append(errs, a.sup.Stop(ctx)) // flushes both queues
}
if a.st != nil {
errs = append(errs, a.st.Close())
}
a.log.Info("stopped")
return errors.Join(errs...)
}
// ensureKey creates the key file on a first run. It refuses when an inventory
// already exists, since a new key could not decrypt its credentials.
func ensureKey(log *slog.Logger, path string, inventoryExists bool) error {
if _, err := os.Stat(path); err == nil {
return nil
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
if inventoryExists {
return fmt.Errorf("secret key file %s is missing but the inventory exists; restore the key file (a new key cannot decrypt the stored SNMP credentials)", path)
}
if err := secrets.GenerateKeyFile(path); err != nil {
return fmt.Errorf("generate secret key: %w", err)
}
log.Error("GENERATED A NEW SECRET KEY: back it up now; without it the SNMP credentials in the inventory cannot be recovered", "path", path)
return nil
}
// inside reports whether path is dir or below it.
func inside(dir, path string) bool {
rel, err := filepath.Rel(resolve(dir), resolve(path))
return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
}
// resolve follows symlinks in the deepest existing parent of p.
func resolve(p string) string {
rest := ""
for q := p; ; q = filepath.Dir(q) {
if r, err := filepath.EvalSymlinks(q); err == nil {
return filepath.Join(r, rest)
}
if filepath.Dir(q) == q {
return p
}
rest = filepath.Join(filepath.Base(q), rest)
}
}
func fileExists(p string) bool {
_, err := os.Stat(p)
return err == nil
}

94
internal/app/app_test.go Normal file
View File

@@ -0,0 +1,94 @@
package app
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/v10networks/nmon/internal/config"
"github.com/v10networks/nmon/internal/secrets"
)
func TestStartRefusals(t *testing.T) {
ctx := context.Background()
data := filepath.Join(t.TempDir(), "data")
outside := filepath.Join(t.TempDir(), "secret.key")
for name, tc := range map[string]struct {
f config.Flags
want string
}{
"no insecure flag": {config.Flags{DataDir: data, SecretKeyFile: outside}, "--insecure-http"},
"no data dir": {config.Flags{InsecureHTTP: true, SecretKeyFile: outside}, "--data-dir"},
"key inside data": {config.Flags{DataDir: data, SecretKeyFile: filepath.Join(data, "keys", "secret.key"), InsecureHTTP: true}, "inside --data-dir"},
"key is data dir": {config.Flags{DataDir: data, SecretKeyFile: data, InsecureHTTP: true}, "inside --data-dir"},
} {
t.Run(name, func(t *testing.T) {
a, err := Start(ctx, tc.f)
if err == nil {
a.Stop(ctx)
t.Fatal("started")
}
if !strings.Contains(err.Error(), tc.want) {
t.Fatalf("error %q, want it to mention %q", err, tc.want)
}
})
}
if _, err := os.Stat(outside); err == nil {
t.Error("a refused start created the key file")
}
}
func TestStartRefusesMissingKeyWithExistingInventory(t *testing.T) {
data := t.TempDir()
if err := os.WriteFile(config.InventoryDB(data), nil, 0o600); err != nil {
t.Fatal(err)
}
key := filepath.Join(t.TempDir(), "secret.key")
_, err := Start(context.Background(), config.Flags{DataDir: data, SecretKeyFile: key, InsecureHTTP: true})
if err == nil || !strings.Contains(err.Error(), "inventory exists") {
t.Fatalf("error %v", err)
}
if _, err := os.Stat(key); err == nil {
t.Error("a new key was generated next to an existing inventory")
}
}
func TestEnsureKeyGeneratesOnce(t *testing.T) {
key := filepath.Join(t.TempDir(), "sub", "secret.key")
if err := ensureKey(testLogger(), key, false); err != nil {
t.Fatal(err)
}
st, err := os.Stat(key)
if err != nil || st.Mode().Perm() != 0o600 {
t.Fatalf("key file: %v %v", st, err)
}
if _, err := secrets.LoadKeyring(key); err != nil {
t.Fatal(err)
}
before, _ := os.ReadFile(key)
if err := ensureKey(testLogger(), key, true); err != nil {
t.Fatal(err)
}
if after, _ := os.ReadFile(key); string(after) != string(before) {
t.Error("existing key was replaced")
}
}
func TestInside(t *testing.T) {
for _, tc := range []struct {
dir, path string
want bool
}{
{"/var/lib/nmon", "/var/lib/nmon/secret.key", true},
{"/var/lib/nmon", "/var/lib/nmon", true},
{"/var/lib/nmon", "/var/lib/nmon2/secret.key", false},
{"/var/lib/nmon", "/etc/nmon/secret.key", false},
{"/var/lib/nmon", "/var/lib/secret.key", false},
} {
if got := inside(tc.dir, tc.path); got != tc.want {
t.Errorf("inside(%s, %s) = %v", tc.dir, tc.path, got)
}
}
}

View File

@@ -0,0 +1,8 @@
package app
import (
"io"
"log/slog"
)
func testLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }

View File

@@ -0,0 +1,175 @@
//go:build integration
package app_test
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"path/filepath"
"sort"
"strconv"
"strings"
"testing"
"time"
"github.com/v10networks/nmon/internal/app"
"github.com/v10networks/nmon/internal/config"
"github.com/v10networks/nmon/internal/simagent"
"github.com/v10networks/nmon/internal/testutil"
"github.com/v10networks/nmon/internal/vmsup"
)
type series struct {
Metric map[string]string `json:"metric"`
Value [2]any `json:"value"`
}
func query(t *testing.T, base, q string) []series {
t.Helper()
resp, err := http.Get(base + "/api/v1/query?q=" + q)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != 200 {
t.Fatalf("query %s: %d %s", q, resp.StatusCode, body)
}
var r struct {
Data struct{ Result []series } `json:"data"`
}
if err := json.Unmarshal(body, &r); err != nil {
t.Fatal(err)
}
return r.Data.Result
}
func TestServeCollectsCountersEndToEnd(t *testing.T) {
rec, err := simagent.LoadRecording("../../testdata/snmprec/counters.snmprec")
if err != nil {
t.Fatal(err)
}
agent := testutil.StartAgent(t, simagent.Config{Recording: rec, Community: "public"})
root := t.TempDir()
data := filepath.Join(root, "data")
a, err := app.Start(context.Background(), config.Flags{
DataDir: data, SecretKeyFile: filepath.Join(root, "keys", "secret.key"), Listen: "[::1]:0", InsecureHTTP: true,
PollInterval: 5 * time.Second, SNMPPort: int(agent.Addr().Port()),
})
if errors.Is(err, vmsup.ErrNoBinary) {
t.Skip("built without the vmembed tag")
}
if err != nil {
t.Fatal(err)
}
stopped := false
stop := func() {
if stopped {
return
}
stopped = true
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
defer cancel()
if err := a.Stop(ctx); err != nil {
t.Errorf("stop: %v", err)
}
}
t.Cleanup(stop)
base := "http://" + a.Addr()
resp, err := http.Post(base+"/api/v1/devices", "application/json", strings.NewReader(
`{"name":"counters","addresses":["::1"],"snmp":{"version":"v2c","community":"public"}}`))
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode != 201 {
t.Fatalf("add device: %d %s", resp.StatusCode, body)
}
var dev struct {
Slug string
Interfaces []struct{ ID string }
}
if err := json.Unmarshal(body, &dev); err != nil || len(dev.Interfaces) != 8 {
t.Fatalf("device: %s (%v)", body, err)
}
// Instant queries return the newest point, so collect values of one counter
// across polls until three distinct ones have been seen.
var vals []float64
deadline := time.Now().Add(60 * time.Second)
for time.Now().Before(deadline) && len(distinct(vals)) < 3 {
time.Sleep(2 * time.Second)
if r, err := http.Post(a.HotURL()+"/internal/force_flush", "", nil); err == nil {
r.Body.Close()
}
for _, s := range query(t, base, "ifHCInOctets") {
if s.Metric["iface"] == dev.Interfaces[0].ID {
v, _ := strconv.ParseFloat(s.Value[1].(string), 64)
vals = append(vals, v)
}
}
}
d := distinct(vals)
if len(d) < 3 {
t.Fatalf("saw only %v for ifHCInOctets after 60s", d)
}
for i := 1; i < len(d); i++ {
if d[i] <= d[i-1] {
t.Fatalf("not rising: %v", d)
}
}
for _, s := range query(t, base, "ifHCInOctets") {
keys := make([]string, 0, len(s.Metric))
for k := range s.Metric {
keys = append(keys, k)
}
sort.Strings(keys)
if fmt.Sprint(keys) != "[__name__ device iface]" || s.Metric["device"] != dev.Slug {
t.Fatalf("labels %v", s.Metric)
}
}
if up := query(t, base, "nmon_snmp_up"); len(up) != 1 || up[0].Value[1] != "1" {
t.Errorf("nmon_snmp_up = %+v", up)
}
// the poller stores nmon_ internals every 15 s
stored := false
for end := time.Now().Add(30 * time.Second); time.Now().Before(end) && !stored; time.Sleep(2 * time.Second) {
if r, err := http.Post(a.HotURL()+"/internal/force_flush", "", nil); err == nil {
r.Body.Close()
}
stored = len(query(t, base, "nmon_pdus_total")) > 0
}
if !stored {
t.Error("nmon_ internals are not stored")
}
if r, err := http.Get(base + "/metrics"); err != nil || r.StatusCode != 200 {
t.Errorf("/metrics: %v %v", r, err)
} else {
b, _ := io.ReadAll(r.Body)
r.Body.Close()
if !bytes.Contains(b, []byte("nmon_pdus_total")) {
t.Error("/metrics lacks nmon_pdus_total")
}
}
stop()
}
func distinct(v []float64) []float64 {
s := append([]float64(nil), v...)
sort.Float64s(s)
out := s[:0]
for i, x := range s {
if i == 0 || x != s[i-1] {
out = append(out, x)
}
}
return out
}

View File

@@ -25,6 +25,7 @@ type Flags struct {
VMLongURL string
MaxInflight int
PollInterval time.Duration // dev/test override of the 60 s interface interval; 0 = profile default
SNMPPort int // dev/test override of UDP 161 for every device; 0 = 161
}
// Config is the content of config.yaml.

View File

@@ -0,0 +1,255 @@
// Package discovery learns what a device has. In M1 it is the seed walk run
// when a device is added: system group plus the IF-MIB tables, creating
// interface rows with ULIDs.
//
// TODO(M2): identity matching beyond ifName and ifDescr+MAC (stack-member
// swaps, RMA), identity_reviews for ambiguous matches, retiring interfaces that
// disappeared, profiles, per-table limits and the `partial` flag.
package discovery
import (
"cmp"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net/netip"
"slices"
"strconv"
"strings"
"time"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/snmp"
"github.com/v10networks/nmon/internal/store"
"github.com/v10networks/nmon/internal/ulid"
)
// Walk roots.
const (
oidSystem = "1.3.6.1.2.1.1"
oidIfTable = "1.3.6.1.2.1.2.2.1"
oidIfXTable = "1.3.6.1.2.1.31.1.1.1"
)
// Target builds the SNMP target for a device's credential. port 0 means 161.
func Target(addr netip.Addr, port int, c *store.SNMPCredential, clk clock.Clock) snmp.Target {
t := snmp.Target{Addr: netip.AddrPortFrom(addr, uint16(port)), Clock: clk}
t.Cred = snmp.Credential{
Community: c.Community, V3User: c.V3User, V3AuthProto: c.V3AuthProto, V3AuthPass: c.V3AuthPass,
V3PrivProto: c.V3PrivProto, V3PrivPass: c.V3PrivPass, V3Context: c.V3Context,
}
t.Cred.Version = strings.TrimPrefix(c.Version, "v")
return t
}
// Iface is one interface as the device reports it.
type Iface struct {
IfIndex int64
Name, Descr string
Alias string
PhysAddr string // colon-separated lower-case hex
Speed int64
HighSpeed int64
Type int64
Admin, Oper int64
HasHC bool
}
// Result is what Collect learned.
type Result struct {
SysDescr, SysObjectID, SysName string
SysUpTime time.Duration
Ifaces []Iface
StartedAt, FinishedAt time.Time
}
// Collect walks the system group and the interface tables over sess, which
// must be a "discovery" session.
func Collect(ctx context.Context, sess *snmp.Session, clk clock.Clock) (*Result, error) {
res := &Result{StartedAt: clk.Now().UTC()}
err := sess.Walk(ctx, oidSystem, 0, func(vb snmp.Varbind) error {
switch vb.OID {
case oidSystem + ".1.0":
res.SysDescr = text(vb)
case oidSystem + ".2.0":
res.SysObjectID, _ = vb.Value.(string)
case oidSystem + ".3.0":
if v, ok := vb.Uint64(); ok {
res.SysUpTime = time.Duration(v) * 10 * time.Millisecond
}
case oidSystem + ".5.0":
res.SysName = text(vb)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("walk system group: %w", err)
}
ifs := map[int64]*Iface{}
get := func(idx int64) *Iface {
i := ifs[idx]
if i == nil {
i = &Iface{IfIndex: idx}
ifs[idx] = i
}
return i
}
for _, c := range []struct {
name string
root string
set func(*Iface, snmp.Varbind)
}{
{"ifIndex", oidIfTable + ".1", func(*Iface, snmp.Varbind) {}},
{"ifDescr", oidIfTable + ".2", func(i *Iface, vb snmp.Varbind) { i.Descr = text(vb) }},
{"ifType", oidIfTable + ".3", func(i *Iface, vb snmp.Varbind) { i.Type = num(vb) }},
{"ifSpeed", oidIfTable + ".5", func(i *Iface, vb snmp.Varbind) { i.Speed = num(vb) }},
{"ifPhysAddress", oidIfTable + ".6", func(i *Iface, vb snmp.Varbind) { i.PhysAddr = mac(vb) }},
{"ifAdminStatus", oidIfTable + ".7", func(i *Iface, vb snmp.Varbind) { i.Admin = num(vb) }},
{"ifOperStatus", oidIfTable + ".8", func(i *Iface, vb snmp.Varbind) { i.Oper = num(vb) }},
{"ifName", oidIfXTable + ".1", func(i *Iface, vb snmp.Varbind) { i.Name = text(vb) }},
{"ifHighSpeed", oidIfXTable + ".15", func(i *Iface, vb snmp.Varbind) { i.HighSpeed = num(vb) }},
{"ifAlias", oidIfXTable + ".18", func(i *Iface, vb snmp.Varbind) { i.Alias = text(vb) }},
{"ifHCInOctets", oidIfXTable + ".6", func(i *Iface, vb snmp.Varbind) { i.HasHC = true }},
} {
prefix := c.root + "."
err := sess.Walk(ctx, c.root, 0, func(vb snmp.Varbind) error {
idx, err := strconv.ParseInt(strings.TrimPrefix(vb.OID, prefix), 10, 64)
if err != nil {
return nil // not a plain single-index row
}
c.set(get(idx), vb)
return nil
})
if err != nil {
return nil, fmt.Errorf("walk %s: %w", c.name, err)
}
}
res.Ifaces = make([]Iface, 0, len(ifs))
for _, i := range ifs {
res.Ifaces = append(res.Ifaces, *i)
}
sortIfaces(res.Ifaces)
res.FinishedAt = clk.Now().UTC()
return res, nil
}
// Seed discovers dev over sess (a "discovery" session) and records the result:
// sys_* fields, interface rows and a discoveries row.
func Seed(ctx context.Context, st store.Store, sess *snmp.Session, dev *store.Device, clk clock.Clock) error {
res, err := Collect(ctx, sess, clk)
if err != nil {
return err
}
return Apply(ctx, st, dev, res, clk)
}
// Apply writes a Collect result for dev. Existing interfaces are matched by
// ifName, then by ifDescr plus MAC, and keep their IDs; the rest get new ULIDs.
// Interfaces the device no longer reports are left alone (TODO(M2): retire or review).
func Apply(ctx context.Context, st store.Store, dev *store.Device, res *Result, clk clock.Clock) error {
return st.WithTx(ctx, func(tx store.Store) error {
existing, err := tx.ListInterfaces(ctx, dev.ID, false)
if err != nil {
return err
}
rows := match(existing, res.Ifaces, clk.Now())
// UpdateDevice writes every mutable column, so start from the stored row
// rather than a caller's copy that may predate SetAddresses.
cur, err := tx.GetDeviceByID(ctx, dev.ID)
if err != nil {
return err
}
cur.SysDescr, cur.SysObjectID, cur.SysName = res.SysDescr, res.SysObjectID, res.SysName
if cur.Profile == "" {
cur.Profile = "generic"
}
if err := tx.UpdateDevice(ctx, cur); err != nil {
return err
}
*dev = *cur
if err := tx.UpsertInterfaces(ctx, dev.ID, rows); err != nil {
return err
}
tables, _ := json.Marshal(map[string]int{"interfaces": len(rows)})
return tx.RecordDiscovery(ctx, &store.Discovery{
DeviceID: dev.ID, StartedAt: res.StartedAt, FinishedAt: res.FinishedAt, TablesJSON: string(tables),
})
})
}
// match pairs discovered interfaces with existing rows and returns the rows to upsert.
func match(existing []store.Interface, found []Iface, now time.Time) []store.Interface {
byName := map[string][]int{}
byDescrMAC := map[string][]int{}
for k, e := range existing {
if e.IfName != "" {
byName[e.IfName] = append(byName[e.IfName], k)
}
key := e.IfDescr + "\x00" + e.IfPhysAddr
byDescrMAC[key] = append(byDescrMAC[key], k)
}
used := make([]bool, len(existing))
take := func(c []int) int {
for _, k := range c {
if !used[k] {
used[k] = true
return k
}
}
return -1
}
out := make([]store.Interface, 0, len(found))
for _, f := range found {
k := -1
if f.Name != "" {
k = take(byName[f.Name])
}
if k < 0 {
k = take(byDescrMAC[f.Descr+"\x00"+f.PhysAddr])
}
var row store.Interface
if k >= 0 {
row = existing[k]
} else {
row = store.Interface{ID: ulid.New(now)}
}
row.IfIndex, row.IfName, row.IfDescr, row.IfAlias = f.IfIndex, f.Name, f.Descr, f.Alias
row.IfPhysAddr, row.IfSpeed, row.IfHighSpeed, row.IfType = f.PhysAddr, f.Speed, f.HighSpeed, f.Type
row.AdminStatus, row.OperStatus, row.HasHCCounters = f.Admin, f.Oper, f.HasHC
out = append(out, row)
}
return out
}
func sortIfaces(s []Iface) {
slices.SortFunc(s, func(a, b Iface) int { return cmp.Compare(a.IfIndex, b.IfIndex) })
}
func text(vb snmp.Varbind) string {
switch v := vb.Value.(type) {
case []byte:
return strings.ToValidUTF8(string(v), "?")
case string:
return v
}
return ""
}
func num(vb snmp.Varbind) int64 {
v, _ := vb.Int64()
return v
}
func mac(vb snmp.Varbind) string {
b, _ := vb.Value.([]byte)
if len(b) == 0 {
return ""
}
h := hex.EncodeToString(b)
parts := make([]string, 0, len(b))
for i := 0; i < len(h); i += 2 {
parts = append(parts, h[i:i+2])
}
return strings.Join(parts, ":")
}

View File

@@ -0,0 +1,115 @@
package discovery_test
import (
"context"
"strconv"
"strings"
"testing"
"time"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/fixturegen"
"github.com/v10networks/nmon/internal/simagent"
"github.com/v10networks/nmon/internal/testutil"
"github.com/v10networks/nmon/internal/ulid"
)
// shuffled returns rec with every interface index i replaced by 100+n+1-i, in
// both ifTable and ifXTable, so every port keeps its name but changes ifIndex.
func shuffled(rec *simagent.Recording, n int) *simagent.Recording {
const ifT, ifX = "1.3.6.1.2.1.2.2.1.", "1.3.6.1.2.1.31.1.1.1."
remap := func(i int) int { return 100 + n + 1 - i }
var out []simagent.Record
for _, r := range rec.Records() {
for _, p := range []string{ifT, ifX} {
if !strings.HasPrefix(r.OID, p) {
continue
}
cut := strings.LastIndexByte(r.OID, '.')
var i int
for _, c := range r.OID[cut+1:] {
i = i*10 + int(c-'0')
}
r.OID = r.OID[:cut+1] + strconv.Itoa(remap(i))
if strings.HasPrefix(r.OID, ifT+"1.") {
r.Value = remap(i)
}
}
out = append(out, r)
}
return simagent.NewRecording(out)
}
func TestSeedCreatesAndKeepsInterfaceIDs(t *testing.T) {
clk := clock.NewFake(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC))
st := testutil.OpenStore(t, clk)
ctx := context.Background()
rec := fixturegen.Generic(fixturegen.Options{Ports: 8, SysName: "sw1", HCCounters: true})
a := testutil.StartAgent(t, simagent.Config{Recording: rec})
dev := testutil.AddDevice(t, st, a, "sw1", "public", clk)
if dev.SysName != "sw1" || dev.SysObjectID != "1.3.6.1.4.1.8072.3.2.10" || dev.SysDescr != "nmon generic fixture" {
t.Fatalf("sys fields: %+v", dev)
}
ifs, err := st.ListInterfaces(ctx, dev.ID, false)
if err != nil || len(ifs) != 8 {
t.Fatalf("interfaces: %d, %v", len(ifs), err)
}
ids := map[string]string{}
for _, i := range ifs {
if !ulid.Valid(i.ID) {
t.Errorf("bad ULID %q", i.ID)
}
if !i.HasHCCounters || i.IfName == "" || i.IfDescr == "" || i.IfSpeed == 0 && i.IfType == 6 {
t.Errorf("incomplete interface %+v", i)
}
ids[i.IfName] = i.ID
}
if ifs[0].IfPhysAddr == "" || strings.Count(ifs[0].IfPhysAddr, ":") != 5 {
t.Errorf("mac %q", ifs[0].IfPhysAddr)
}
d, err := st.LastDiscovery(ctx, dev.ID)
if err != nil || d.Partial || d.TablesJSON == "{}" {
t.Fatalf("discovery row: %+v %v", d, err)
}
// Same device again with every ifIndex shuffled: IDs must follow the ifName.
a2 := testutil.StartAgent(t, simagent.Config{Recording: shuffled(rec, 8)})
cred, err := st.GetCredential(ctx, *dev.SNMPCredentialID)
if err != nil {
t.Fatal(err)
}
testutil.Seed(t, st, a2, dev, cred, clk)
after, err := st.ListInterfaces(ctx, dev.ID, false)
if err != nil || len(after) != 8 {
t.Fatalf("after reshuffle: %d, %v", len(after), err)
}
moved := 0
for _, i := range after {
if ids[i.IfName] != i.ID {
t.Errorf("%s: id %s, was %s", i.IfName, i.ID, ids[i.IfName])
}
if i.IfIndex > 100 {
moved++
}
}
if moved != 8 {
t.Errorf("%d interfaces carry the new ifIndex, want 8", moved)
}
}
func TestSeedWithoutHCCounters(t *testing.T) {
clk := clock.NewFake(time.Now())
st := testutil.OpenStore(t, clk)
a := testutil.StartAgent(t, simagent.Config{Recording: fixturegen.Generic(fixturegen.Options{Ports: 4})})
dev := testutil.AddDevice(t, st, a, "old", "public", clk)
ifs, _ := st.ListInterfaces(context.Background(), dev.ID, false)
if len(ifs) != 4 {
t.Fatalf("%d interfaces", len(ifs))
}
for _, i := range ifs {
if i.HasHCCounters {
t.Errorf("%s: has_hc_counters set", i.IfName)
}
}
}

299
internal/poller/cycle.go Normal file
View File

@@ -0,0 +1,299 @@
package poller
import (
"context"
"errors"
"fmt"
"net/netip"
"github.com/v10networks/nmon/internal/discovery"
"github.com/v10networks/nmon/internal/icmp"
"github.com/v10networks/nmon/internal/metrics"
"github.com/v10networks/nmon/internal/store"
)
// kind says how a varbind becomes a sample value.
type kind int
const (
kindUptime kind = iota // TimeTicks to seconds
kindCounter // unsigned
kindStatus // integer enum
)
// item is what one requested OID means.
type item struct {
name string // metric name
iface string // interface ULID; empty for device-level series
kind kind
}
// target is a device as the loop polls it; it is replaced wholesale on reload.
type target struct {
dev store.Device
addr netip.Addr
cred *store.SNMPCredential // nil: the device has no SNMP credential
oids []string
items []item
ifaces int
}
const oidSysUpTime = "1.3.6.1.2.1.1.3.0"
// IF-MIB columns polled every cycle: ifTable under 1.3.6.1.2.1.2.2.1, ifXTable under 1.3.6.1.2.1.31.1.1.1.
var (
hcColumns = []struct {
name, col string
kind kind
}{
{"ifHCInOctets", "1.3.6.1.2.1.31.1.1.1.6", kindCounter},
{"ifHCOutOctets", "1.3.6.1.2.1.31.1.1.1.10", kindCounter},
}
// Without HC counters the 32-bit octet counters are read but keep the
// non-HC names, so a series never silently changes width under one name.
plainColumns = []struct {
name, col string
kind kind
}{
{"ifInOctets", "1.3.6.1.2.1.2.2.1.10", kindCounter},
{"ifOutOctets", "1.3.6.1.2.1.2.2.1.16", kindCounter},
}
commonColumns = []struct {
name, col string
kind kind
}{
{"ifInErrors", "1.3.6.1.2.1.2.2.1.14", kindCounter},
{"ifOutErrors", "1.3.6.1.2.1.2.2.1.20", kindCounter},
{"ifInDiscards", "1.3.6.1.2.1.2.2.1.13", kindCounter},
{"ifOutDiscards", "1.3.6.1.2.1.2.2.1.19", kindCounter},
{"ifOperStatus", "1.3.6.1.2.1.2.2.1.8", kindStatus},
{"ifAdminStatus", "1.3.6.1.2.1.2.2.1.7", kindStatus},
}
)
func (p *Poller) buildTarget(ctx context.Context, d *store.Device) (*target, error) {
t := &target{dev: *d}
if a, err := netip.ParseAddr(d.MgmtIP); err == nil {
t.addr = a
}
if d.SNMPCredentialID != nil {
c, err := p.o.Store.GetCredential(ctx, *d.SNMPCredentialID)
if err != nil {
return nil, fmt.Errorf("credential: %w", err)
}
t.cred = c
ifs, err := p.o.Store.ListInterfaces(ctx, d.ID, false)
if err != nil {
return nil, fmt.Errorf("interfaces: %w", err)
}
t.oids = append(t.oids, oidSysUpTime)
t.items = append(t.items, item{name: "sysUpTime", kind: kindUptime})
for _, i := range ifs {
if i.Excluded || i.RetiredAt != nil {
continue
}
t.ifaces++
octets := plainColumns
if i.HasHCCounters {
octets = hcColumns
}
for _, cols := range [][]struct {
name, col string
kind kind
}{octets, commonColumns} {
for _, c := range cols {
t.oids = append(t.oids, fmt.Sprintf("%s.%d", c.col, i.IfIndex))
t.items = append(t.items, item{name: c.name, iface: i.ID, kind: c.kind})
}
}
}
}
return t, nil
}
// runSNMP polls the device every interval, starting at a stable per-device offset.
func (l *devLoop) runSNMP(ctx context.Context) {
p := l.p
off := offset
if p.o.Offset != nil {
off = p.o.Offset
}
next := p.clk.Now().Add(off(l.slug, p.o.Interval))
for sleepUntil(ctx, p.clk, next) {
l.cycle(ctx)
next = advance(next, p.clk.Now(), p.o.Interval)
}
}
// cycle runs one poll under the cycle deadline. It returns only when the poll
// has finished or been abandoned, so cycles of one device never overlap.
func (l *devLoop) cycle(ctx context.Context) {
p := l.p
t := l.tgt.Load()
if t == nil || t.cred == nil || !t.addr.IsValid() {
return
}
start := p.clk.Now()
cctx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
deadline := p.clk.After(p.o.Interval / 2) // registered before polling starts
go func() {
select {
case <-deadline:
cancel(errDeadline)
case <-cctx.Done():
}
}()
samples, skipped, err := l.poll(cctx, t)
if ctx.Err() != nil {
return // shutting down
}
res := CycleResult{Device: l.slug, Err: err, Samples: len(samples), Skipped: skipped}
if err != nil && errors.Is(context.Cause(cctx), errDeadline) {
res.Missed = true
l.missed.Inc()
}
now := p.clk.Now()
l.duration.Set(now.Sub(start).Seconds())
l.skipped.Add(float64(skipped))
up := 0.0
if err == nil {
up = 1
if terr := p.o.Store.TouchDevice(ctx, t.dev.ID, now.UTC()); terr != nil {
p.log.Warn("touch device", "device", l.slug, "err", terr)
}
}
samples = append(samples, metrics.New("nmon_snmp_up", up, now, metrics.LabelDevice, l.slug))
if werr := p.o.Writer.Write(ctx, samples); werr != nil {
p.log.Warn("write samples", "device", l.slug, "err", werr)
}
if l.snmpUp == nil || *l.snmpUp != (err == nil) {
ok := err == nil
l.snmpUp = &ok
if ok {
p.log.Info("snmp up", "device", l.slug)
} else {
p.log.Warn("snmp down", "device", l.slug, "err", err, "deadline_missed", res.Missed)
}
}
if p.o.OnCycle != nil {
p.o.OnCycle(res)
}
}
// poll does one multi-varbind GET and converts the answer to samples.
func (l *devLoop) poll(ctx context.Context, t *target) ([]metrics.Sample, int, error) {
p := l.p
select {
case p.sem <- struct{}{}:
defer func() { <-p.sem }()
case <-ctx.Done():
return nil, 0, ctx.Err()
}
tg := discovery.Target(t.addr, p.o.SNMPPort, t.cred, p.clk)
sess, err := p.o.Dial(ctx, tg)
if err != nil {
return nil, 0, err
}
defer sess.Close()
vbs, err := sess.Get(ctx, t.oids)
if err != nil {
return nil, 0, err
}
if len(vbs) != len(t.oids) {
return nil, 0, fmt.Errorf("asked for %d varbinds, got %d", len(t.oids), len(vbs))
}
ts := p.clk.Now()
out := make([]metrics.Sample, 0, len(vbs)+1)
skipped := 0
for i, vb := range vbs {
it := t.items[i]
if vb.Exception() || vb.OID != t.oids[i] {
skipped++
continue
}
var v float64
switch it.kind {
case kindUptime:
u, ok := vb.Uint64()
if !ok {
skipped++
continue
}
v = float64(u) / 100
case kindCounter:
u, ok := vb.Uint64()
if !ok {
skipped++
continue
}
v = float64(u)
case kindStatus:
n, ok := vb.Int64()
if !ok {
skipped++
continue
}
v = float64(n)
}
if it.iface == "" {
out = append(out, metrics.New(it.name, v, ts, metrics.LabelDevice, l.slug))
} else {
out = append(out, metrics.New(it.name, v, ts, metrics.LabelDevice, l.slug, metrics.LabelIface, it.iface))
}
}
return out, skipped, nil
}
// runICMP probes the preferred management address every 15 s. An unsupported
// probe writes nothing and stops: it is unknown, never down.
func (l *devLoop) runICMP(ctx context.Context) {
p := l.p
next := p.clk.Now().Add(offset(l.slug, icmpInterval))
var (
known, up bool
hits, misses int
)
for sleepUntil(ctx, p.clk, next) {
next = advance(next, p.clk.Now(), icmpInterval)
t := l.tgt.Load()
if t == nil || !t.addr.IsValid() {
continue
}
rtt, err := p.o.Pinger.Ping(ctx, t.addr, icmpTimeout)
if ctx.Err() != nil {
return
}
if errors.Is(err, icmp.ErrUnsupported) {
p.log.Warn("icmp probe unsupported; not probing", "device", l.slug, "err", err)
return
}
now := p.clk.Now()
var out []metrics.Sample
if err == nil {
hits, misses = hits+1, 0
if !known || (!up && hits >= icmpUpAfter) {
known, up = true, true
}
out = append(out, metrics.New("nmon_rtt_seconds", rtt.Seconds(), now, metrics.LabelDevice, l.slug))
} else {
hits, misses = 0, misses+1
if misses >= icmpDownAfter && (!known || up) {
known, up = true, false
}
}
if known {
v := 0.0
if up {
v = 1
}
out = append(out, metrics.New("nmon_icmp_up", v, now, metrics.LabelDevice, l.slug))
}
if len(out) > 0 {
if werr := p.o.Writer.Write(ctx, out); werr != nil {
p.log.Warn("write icmp samples", "device", l.slug, "err", werr)
}
}
}
}

276
internal/poller/poller.go Normal file
View File

@@ -0,0 +1,276 @@
// Package poller runs the 60 s collection path: a jittered, deadline-bounded
// SNMP GET of the generic interface counters per device, and an ICMP probe
// every 15 s. Each device has exactly one SNMP goroutine, so a device is never
// polled by two cycles at once; a global semaphore caps in-flight polls.
//
// TODO(M2): profiles instead of the hard-coded metric list, varbind
// adaptation on tooBig, backoff and the degraded state, the 30 s cycle for
// devices without HC counters.
package poller
import (
"context"
"errors"
"hash/fnv"
"log/slog"
"net/netip"
"sync"
"sync/atomic"
"time"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/icmp"
"github.com/v10networks/nmon/internal/metrics"
"github.com/v10networks/nmon/internal/selfmon"
"github.com/v10networks/nmon/internal/snmp"
"github.com/v10networks/nmon/internal/store"
)
// Defaults.
const (
DefaultInterval = 60 * time.Second
DefaultMaxInflight = 512
icmpInterval = 15 * time.Second
selfmonInterval = 15 * time.Second
icmpTimeout = 2 * time.Second
icmpDownAfter = 3 // consecutive misses
icmpUpAfter = 2 // consecutive hits after being down
)
// Session is the part of *snmp.Session the poller uses.
type Session interface {
Get(ctx context.Context, oids []string) ([]snmp.Varbind, error)
Close() error
}
// Dialer opens a poll-path session. The default is snmp.Dial with path "poll".
type Dialer func(ctx context.Context, t snmp.Target) (Session, error)
// Pinger sends one echo request. *icmp.Pinger implements it.
type Pinger interface {
Ping(ctx context.Context, addr netip.Addr, timeout time.Duration) (time.Duration, error)
}
// CycleResult describes one finished SNMP cycle of one device.
type CycleResult struct {
Device string
Err error // nil on success
Missed bool // the cycle deadline passed before the poll finished
Samples int // interface and sysUpTime samples written
Skipped int // varbinds skipped: exceptions or non-numeric values
}
// Options configures a Poller. Store and Writer are required.
type Options struct {
Store store.Store
Writer metrics.Writer // the hot write queue
Clock clock.Clock // default clock.Real()
Interval time.Duration // per-device SNMP interval; default 60 s. The cycle deadline is half of it
MaxInflight int // global cap on concurrent polls; default 512
SNMPPort int // dev/test override of UDP 161; 0 = 161
Registry *selfmon.Registry // default selfmon.Default
Logger *slog.Logger // default slog.Default()
// Test hooks.
Offset func(slug string, interval time.Duration) time.Duration // first-poll offset; default stable hash
Dial Dialer
Pinger Pinger
OnCycle func(CycleResult)
}
// Poller schedules and runs all device polls.
type Poller struct {
o Options
clk clock.Clock
log *slog.Logger
reg *selfmon.Registry
sem chan struct{}
refresh chan struct{}
started chan struct{}
once sync.Once
loops map[string]*devLoop // owned by Run
wg sync.WaitGroup
}
// New returns a Poller; nothing runs until Run.
func New(o Options) *Poller {
if o.Clock == nil {
o.Clock = clock.Real()
}
if o.Interval <= 0 {
o.Interval = DefaultInterval
}
if o.MaxInflight <= 0 {
o.MaxInflight = DefaultMaxInflight
}
if o.Registry == nil {
o.Registry = selfmon.Default
}
if o.Logger == nil {
o.Logger = slog.Default()
}
if o.Dial == nil {
o.Dial = func(ctx context.Context, t snmp.Target) (Session, error) {
s, err := snmp.Dial(ctx, t, "poll")
if err != nil {
return nil, err
}
return s, nil
}
}
if o.Pinger == nil {
o.Pinger = icmp.New(o.Clock)
}
return &Poller{
o: o, clk: o.Clock, log: o.Logger.With("component", "poller"), reg: o.Registry,
sem: make(chan struct{}, o.MaxInflight),
refresh: make(chan struct{}, 1),
started: make(chan struct{}),
loops: map[string]*devLoop{},
}
}
// Refresh asks the poller to reload its device list from the store. It never blocks.
func (p *Poller) Refresh() {
select {
case p.refresh <- struct{}{}:
default:
}
}
// Started is closed once the first device list has been loaded and its loops started.
func (p *Poller) Started() <-chan struct{} { return p.started }
// Run polls until ctx ends, then waits for every loop to stop.
func (p *Poller) Run(ctx context.Context) error {
defer p.wg.Wait()
if err := p.reload(ctx); err != nil {
p.log.Error("load devices", "err", err)
}
p.once.Do(func() { close(p.started) })
tick := p.clk.NewTicker(selfmonInterval)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-p.refresh:
if err := p.reload(ctx); err != nil {
p.log.Error("reload devices", "err", err)
}
case <-tick.C():
now := p.clk.Now()
if err := p.o.Writer.Write(ctx, p.reg.Snapshot(now)); err != nil {
p.log.Warn("write nmon_ metrics", "err", err)
}
}
}
}
// reload syncs the device loops with the store.
func (p *Poller) reload(ctx context.Context) error {
var devs []store.Device
for off := int64(0); ; {
page, total, err := p.o.Store.ListDevices(ctx, store.DeviceFilter{}, store.Page{Offset: off, Limit: store.MaxLimit})
if err != nil {
return err
}
devs = append(devs, page...)
off += int64(len(page))
if len(page) == 0 || off >= total {
break
}
}
seen := make(map[string]bool, len(devs))
for i := range devs {
d := &devs[i]
if !d.Enabled || d.RetiredAt != nil {
continue
}
t, err := p.buildTarget(ctx, d)
if err != nil {
p.log.Error("load device", "device", d.Slug, "err", err)
if l := p.loops[d.Slug]; l != nil {
seen[d.Slug] = true // keep polling with the previous settings
}
continue
}
seen[d.Slug] = true
l := p.loops[d.Slug]
if l == nil {
l = p.startLoop(ctx, d.Slug)
}
l.tgt.Store(t)
}
for slug, l := range p.loops {
if !seen[slug] {
l.cancel()
delete(p.loops, slug)
}
}
return nil
}
func (p *Poller) startLoop(ctx context.Context, slug string) *devLoop {
lctx, cancel := context.WithCancel(ctx)
l := &devLoop{
p: p, slug: slug, cancel: cancel,
duration: p.reg.Gauge("nmon_poll_duration_seconds", "device", slug),
missed: p.reg.Counter("nmon_poll_deadline_missed_total", "device", slug),
skipped: p.reg.Counter("nmon_poll_skipped_varbinds_total", "device", slug),
}
p.loops[slug] = l
p.wg.Add(2)
go func() { defer p.wg.Done(); l.runSNMP(lctx) }()
go func() { defer p.wg.Done(); l.runICMP(lctx) }()
return l
}
// offset spreads first polls uniformly over the interval, stably per device.
func offset(slug string, interval time.Duration) time.Duration {
h := fnv.New64a()
h.Write([]byte(slug))
return time.Millisecond + time.Duration(h.Sum64()%uint64(interval-time.Millisecond))
}
// sleepUntil waits for the clock to reach t; it returns false if ctx ended first.
func sleepUntil(ctx context.Context, clk clock.Clock, t time.Time) bool {
d := t.Sub(clk.Now())
if d <= 0 {
return ctx.Err() == nil
}
select {
case <-ctx.Done():
return false
case <-clk.After(d):
return true
}
}
// advance returns the first scheduled time after now.
func advance(next, now time.Time, every time.Duration) time.Time {
next = next.Add(every)
for !next.After(now) {
next = next.Add(every)
}
return next
}
var errDeadline = errors.New("poller: cycle deadline")
// devLoop is the pair of goroutines serving one device.
type devLoop struct {
p *Poller
slug string
cancel context.CancelFunc
tgt atomic.Pointer[target]
duration selfmon.Gauge
missed selfmon.Counter
skipped selfmon.Counter
snmpUp *bool // last state logged; touched only by the SNMP goroutine
}

View File

@@ -0,0 +1,604 @@
package poller_test
import (
"context"
"errors"
"net/netip"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/gosnmp/gosnmp"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/fixturegen"
"github.com/v10networks/nmon/internal/icmp"
"github.com/v10networks/nmon/internal/metrics"
"github.com/v10networks/nmon/internal/poller"
"github.com/v10networks/nmon/internal/selfmon"
"github.com/v10networks/nmon/internal/simagent"
"github.com/v10networks/nmon/internal/snmp"
"github.com/v10networks/nmon/internal/store"
"github.com/v10networks/nmon/internal/testutil"
"github.com/v10networks/nmon/internal/ulid"
)
const interval = 60 * time.Second
var t0 = time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
type writer struct {
mu sync.Mutex
all []metrics.Sample
}
func (w *writer) Write(_ context.Context, s []metrics.Sample) error {
w.mu.Lock()
w.all = append(w.all, s...)
w.mu.Unlock()
return nil
}
func (w *writer) samples(name string) []metrics.Sample {
w.mu.Lock()
defer w.mu.Unlock()
var out []metrics.Sample
for _, s := range w.all {
if s.Name == name {
out = append(out, s)
}
}
return out
}
func (w *writer) snapshot() []metrics.Sample {
w.mu.Lock()
defer w.mu.Unlock()
return append([]metrics.Sample(nil), w.all...)
}
type pinger struct {
mu sync.Mutex
results []error // consumed in order; the last one repeats
calls int
}
func (p *pinger) Ping(context.Context, netip.Addr, time.Duration) (time.Duration, error) {
p.mu.Lock()
defer p.mu.Unlock()
var err error
if len(p.results) > 0 {
err = p.results[min(p.calls, len(p.results)-1)]
}
p.calls++
return 3 * time.Millisecond, err
}
func (p *pinger) count() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls }
type harness struct {
t *testing.T
clk *clock.Fake
st store.Store
w *writer
p *poller.Poller
cycles chan poller.CycleResult
ping *pinger
cancel context.CancelFunc
done chan struct{}
counts map[string]int
}
type opts struct {
maxInflight int
dial poller.Dialer
offset func(string, time.Duration) time.Duration
}
func newHarness(t *testing.T, clk *clock.Fake, st store.Store, o opts) *harness {
h := &harness{t: t, clk: clk, st: st, w: &writer{}, cycles: make(chan poller.CycleResult, 4096), ping: &pinger{}, counts: map[string]int{}}
h.p = poller.New(poller.Options{
Store: st, Writer: h.w, Clock: clk, Interval: interval, MaxInflight: o.maxInflight,
Dial: o.dial, Pinger: h.ping, Offset: o.offset,
OnCycle: func(r poller.CycleResult) { h.cycles <- r },
})
return h
}
// start runs the poller and waits until every loop is blocked on the clock.
func (h *harness) start(devices int) {
ctx, cancel := context.WithCancel(context.Background())
h.cancel = cancel
h.done = make(chan struct{})
go func() { h.p.Run(ctx); close(h.done) }()
h.t.Cleanup(h.stop)
<-h.p.Started()
// per device: SNMP and ICMP sleeps; plus the nmon_ metrics ticker
eventually(h.t, func() bool { return h.clk.Waiters() >= 2*devices+1 })
}
func (h *harness) stop() {
h.cancel()
<-h.done
}
// drain collects finished cycles for up to wait.
func (h *harness) drain(wait time.Duration) []poller.CycleResult {
var out []poller.CycleResult
for {
select {
case r := <-h.cycles:
h.counts[r.Device]++
out = append(out, r)
case <-time.After(wait):
return out
}
}
}
// until advances the fake clock in quarter-interval steps until cond holds.
func (h *harness) until(cond func() bool) {
h.t.Helper()
for i := 0; i < 40 && !cond(); i++ {
h.clk.Advance(interval / 4)
h.drain(25 * time.Millisecond)
}
if !cond() {
h.t.Fatal("condition not reached after advancing the clock")
}
}
func eventually(t *testing.T, cond func() bool) {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatal("timed out waiting for condition")
}
time.Sleep(2 * time.Millisecond)
}
}
// fixedOffset starts every device 1 s into the interval, so all cycles share one deadline window.
func fixedOffset(string, time.Duration) time.Duration { return time.Second }
func counter(name string, kv ...string) float64 {
return selfmon.Default.Counter(name, kv...).Value()
}
func TestPollWritesExactLabelsAndRisingValues(t *testing.T) {
clk := clock.NewFake(t0)
st := testutil.OpenStore(t, clk)
a := testutil.StartAgent(t, simagent.Config{
Recording: fixturegen.Generic(fixturegen.Options{Ports: 8, HCCounters: true, Counters: true}), Clock: clk,
})
dev := testutil.AddDevice(t, st, a, "sw1", "public", clk)
ifs, _ := st.ListInterfaces(context.Background(), dev.ID, false)
ids := map[string]bool{}
for _, i := range ifs {
ids[i.ID] = true
}
a.ResetStats()
pdus := func(typ string) float64 { return counter("nmon_pdus_total", "path", "poll", "type", typ) }
get0, next0, bulk0 := pdus("get"), pdus("getnext"), pdus("getbulk")
h := newHarness(t, clk, st, opts{})
h.p = poller.New(poller.Options{
Store: st, Writer: h.w, Clock: clk, Interval: interval, Pinger: h.ping, SNMPPort: int(a.Addr().Port()),
OnCycle: func(r poller.CycleResult) { h.cycles <- r },
})
h.start(1)
h.until(func() bool { return h.counts[dev.Slug] >= 3 })
time.Sleep(100 * time.Millisecond) // let any cycle in flight finish; the clock no longer moves
h.drain(10 * time.Millisecond)
n := h.counts[dev.Slug]
names := map[string]bool{}
perIface := map[string][]float64{}
for _, s := range h.w.snapshot() {
if len(s.Name) < 3 || s.Name[:3] == "nmo" && s.Name != "nmon_snmp_up" {
continue // selfmon snapshots and icmp are checked elsewhere
}
names[s.Name] = true
switch s.Name {
case "sysUpTime", "nmon_snmp_up":
if len(s.Labels) != 1 || s.Labels[0] != (metrics.Label{Name: "device", Value: dev.Slug}) {
t.Fatalf("%s labels %v", s.Name, s.Labels)
}
if s.Name == "nmon_snmp_up" && s.Value != 1 {
t.Fatalf("snmp_up %v", s.Value)
}
default:
if len(s.Labels) != 2 || s.Labels[0].Name != "device" || s.Labels[0].Value != dev.Slug ||
s.Labels[1].Name != "iface" || !ids[s.Labels[1].Value] {
t.Fatalf("%s labels %v", s.Name, s.Labels)
}
if s.Name == "ifHCInOctets" {
perIface[s.Labels[1].Value] = append(perIface[s.Labels[1].Value], s.Value)
}
}
}
want := []string{"sysUpTime", "nmon_snmp_up", "ifHCInOctets", "ifHCOutOctets", "ifInErrors", "ifOutErrors",
"ifInDiscards", "ifOutDiscards", "ifOperStatus", "ifAdminStatus"}
if len(names) != len(want) {
t.Errorf("metric names %v", names)
}
for _, w := range want {
if !names[w] {
t.Errorf("missing %s", w)
}
}
if got := len(h.w.samples("ifHCInOctets")); got != 8*n {
t.Errorf("%d ifHCInOctets samples for %d cycles of 8 ports", got, n)
}
for id, vs := range perIface {
for i := 1; i < len(vs); i++ {
if vs[i] <= vs[i-1] {
t.Errorf("iface %s: %v not rising", id, vs)
}
}
}
if up := h.w.samples("sysUpTime"); len(up) != n || up[1].Value <= up[0].Value {
t.Errorf("sysUpTime %v", up)
}
// 1 sysUpTime + 8 ports x 8 objects = 65 varbinds = 2 PDUs of at most 40, all GET.
st1 := a.Stats()
if st1.Get != uint64(2*n) || st1.GetNext != 0 || st1.GetBulk != 0 {
t.Errorf("agent saw %+v for %d cycles, want %d GETs and no walks", st1, n, 2*n)
}
if d := pdus("get") - get0; d != float64(2*n) {
t.Errorf("nmon_pdus_total{poll,get} rose by %v, want %d", d, 2*n)
}
if pdus("getnext") != next0 || pdus("getbulk") != bulk0 {
t.Error("walk PDUs counted on the poll path")
}
if got, _ := st.GetDeviceBySlug(context.Background(), dev.Slug); got.LastSeen == nil {
t.Error("last_seen not updated")
}
}
func TestPollWithoutHCUsesPlainNames(t *testing.T) {
clk := clock.NewFake(t0)
st := testutil.OpenStore(t, clk)
a := testutil.StartAgent(t, simagent.Config{Recording: fixturegen.Generic(fixturegen.Options{Ports: 4, Counters: true}), Clock: clk})
dev := testutil.AddDevice(t, st, a, "old", "public", clk)
h := newHarness(t, clk, st, opts{})
h.p = poller.New(poller.Options{Store: st, Writer: h.w, Clock: clk, Interval: interval, Pinger: h.ping,
SNMPPort: int(a.Addr().Port()), OnCycle: func(r poller.CycleResult) { h.cycles <- r }})
h.start(1)
h.until(func() bool { return h.counts[dev.Slug] >= 1 })
if len(h.w.samples("ifInOctets")) != 4 || len(h.w.samples("ifOutOctets")) != 4 {
t.Errorf("want 4 ifInOctets and ifOutOctets, got %d/%d", len(h.w.samples("ifInOctets")), len(h.w.samples("ifOutOctets")))
}
if len(h.w.samples("ifHCInOctets")) != 0 || len(h.w.samples("ifHCOutOctets")) != 0 {
t.Error("HC names used for a device without HC counters")
}
}
// fakeDevices creates n SNMP devices with two interfaces each, no agent behind them.
func fakeDevices(t *testing.T, st store.Store, n int) []*store.Device {
ctx := context.Background()
var out []*store.Device
for i := 0; i < n; i++ {
name := "dev" + string(rune('a'+i))
cred := &store.SNMPCredential{Name: name, Version: "v2c", Community: name}
if err := st.CreateCredential(ctx, cred); err != nil {
t.Fatal(err)
}
d := &store.Device{Name: name, Kind: store.KindNetwork, SNMPCredentialID: &cred.ID, Enabled: true, Profile: "generic"}
if err := st.CreateDevice(ctx, d); err != nil {
t.Fatal(err)
}
if err := st.SetAddresses(ctx, d.ID, []string{"::1"}); err != nil {
t.Fatal(err)
}
var ifs []store.Interface
for j := 1; j <= 2; j++ {
ifs = append(ifs, store.Interface{ID: ulid.New(t0), IfIndex: int64(j), IfName: "p" + string(rune('0'+j)), HasHCCounters: true})
}
if err := st.UpsertInterfaces(ctx, d.ID, ifs); err != nil {
t.Fatal(err)
}
out = append(out, d)
}
return out
}
// blockingSession answers every Get with ones after release, or fails with ctx.Err().
type blocker struct {
mu sync.Mutex
cur map[string]int
max map[string]int
global atomic.Int32
gmax atomic.Int32
release chan struct{}
dialed atomic.Int32
}
func newBlocker() *blocker {
return &blocker{cur: map[string]int{}, max: map[string]int{}, release: make(chan struct{})}
}
type blockSession struct {
b *blocker
comm string
}
func (b *blocker) dial(_ context.Context, tg snmp.Target) (poller.Session, error) {
b.dialed.Add(1)
return &blockSession{b, tg.Cred.Community}, nil
}
func (s *blockSession) Get(ctx context.Context, oids []string) ([]snmp.Varbind, error) {
b := s.b
b.mu.Lock()
b.cur[s.comm]++
b.max[s.comm] = max(b.max[s.comm], b.cur[s.comm])
b.mu.Unlock()
g := b.global.Add(1)
for {
old := b.gmax.Load()
if g <= old || b.gmax.CompareAndSwap(old, g) {
break
}
}
defer func() {
b.global.Add(-1)
b.mu.Lock()
b.cur[s.comm]--
b.mu.Unlock()
}()
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-b.release:
}
out := make([]snmp.Varbind, len(oids))
for i, o := range oids {
out[i] = snmp.Varbind{OID: o, Type: gosnmp.Integer, Value: 1}
}
return out, nil
}
func (s *blockSession) Close() error { return nil }
func TestPerDeviceConcurrencyIsOneEvenWhenDeadlineBeatsLatency(t *testing.T) {
clk := clock.NewFake(t0)
st := testutil.OpenStore(t, clk)
devs := fakeDevices(t, st, 3)
b := newBlocker()
missed0 := map[string]float64{}
for _, d := range devs {
missed0[d.Slug] = counter("nmon_poll_deadline_missed_total", "device", d.Slug)
}
h := newHarness(t, clk, st, opts{dial: b.dial})
h.start(3)
h.until(func() bool {
for _, d := range devs {
if h.counts[d.Slug] < 3 {
return false
}
}
return true
})
b.mu.Lock()
defer b.mu.Unlock()
for _, d := range devs {
if m := b.max[d.Name]; m != 1 {
t.Errorf("%s: max concurrent polls %d, want 1", d.Slug, m)
}
got := counter("nmon_poll_deadline_missed_total", "device", d.Slug) - missed0[d.Slug]
if got < 3 || got != float64(h.counts[d.Slug]) {
t.Errorf("%s: deadline missed %v times in %d cycles", d.Slug, got, h.counts[d.Slug])
}
if dur := selfmon.Default.Gauge("nmon_poll_duration_seconds", "device", d.Slug).Value(); dur < 30 {
t.Errorf("%s: poll duration %vs, want at least the 30s deadline", d.Slug, dur)
}
}
for _, s := range h.w.samples("nmon_snmp_up") {
if s.Value != 0 {
t.Fatalf("snmp_up %v after a missed deadline", s.Value)
}
}
}
func TestGlobalInflightCap(t *testing.T) {
clk := clock.NewFake(t0)
st := testutil.OpenStore(t, clk)
devs := fakeDevices(t, st, 5)
b := newBlocker()
h := newHarness(t, clk, st, opts{dial: b.dial, maxInflight: 2, offset: fixedOffset})
h.start(5)
clk.Advance(2 * time.Second) // every device starts its cycle
eventually(t, func() bool { return b.global.Load() == 2 })
time.Sleep(100 * time.Millisecond)
if g := b.gmax.Load(); g != 2 {
t.Fatalf("max in flight %d with --max-inflight 2", g)
}
close(b.release)
got := map[string]bool{}
deadline := time.After(10 * time.Second)
for len(got) < len(devs) {
select {
case r := <-h.cycles:
if r.Err != nil {
t.Fatalf("%s: %v", r.Device, r.Err)
}
got[r.Device] = true
case <-deadline:
t.Fatalf("only %d of %d devices finished", len(got), len(devs))
}
}
if g := b.gmax.Load(); g != 2 {
t.Fatalf("max in flight %d after release", g)
}
}
func TestSlowDeviceDoesNotDelayOthers(t *testing.T) {
clk := clock.NewFake(t0)
st := testutil.OpenStore(t, clk)
rec := fixturegen.Generic(fixturegen.Options{Ports: 8, HCCounters: true, Counters: true})
ports := map[string]uint16{}
normal := map[string]bool{}
for i := 0; i < 10; i++ {
name := "norm" + string(rune('a'+i))
a := testutil.StartAgent(t, simagent.Config{Recording: rec, Clock: clk, Community: name})
d := testutil.AddDevice(t, st, a, name, name, clk)
ports[name] = a.Addr().Port()
normal[d.Slug] = true
}
// The slow device is seeded against a fast agent, then replaced by one that answers after 3 s.
fast := testutil.StartAgent(t, simagent.Config{Recording: rec, Clock: clk, Community: "slow"})
slow := testutil.AddDevice(t, st, fast, "slow", "slow", clk)
fast.Close()
s := testutil.StartAgent(t, simagent.Config{Recording: rec, Community: "slow", Behaviour: simagent.Behaviour{Latency: 3 * time.Second}})
ports["slow"] = s.Addr().Port()
var slowDialed atomic.Bool
dial := func(ctx context.Context, tg snmp.Target) (poller.Session, error) {
tg.Addr = netip.AddrPortFrom(tg.Addr.Addr(), ports[tg.Cred.Community])
if tg.Cred.Community == "slow" {
defer slowDialed.Store(true)
}
sess, err := snmp.Dial(ctx, tg, "poll")
if err != nil {
return nil, err
}
return sess, nil
}
missed0 := counter("nmon_poll_deadline_missed_total", "device", slow.Slug)
h := newHarness(t, clk, st, opts{dial: dial, offset: fixedOffset})
h.start(11)
clk.Advance(2 * time.Second) // one tick: every device starts
done := map[string]bool{}
deadline := time.After(10 * time.Second)
for len(done) < len(normal) {
select {
case r := <-h.cycles:
if r.Device == slow.Slug {
t.Fatalf("slow device finished early: %+v", r)
}
if r.Err != nil {
t.Fatalf("%s: %v", r.Device, r.Err)
}
done[r.Device] = true
case <-deadline:
t.Fatalf("only %d of %d normal devices finished within the first tick", len(done), len(normal))
}
}
eventually(t, slowDialed.Load)
if got := counter("nmon_poll_deadline_missed_total", "device", slow.Slug); got != missed0 {
t.Fatalf("deadline counted before it passed: %v", got)
}
clk.Advance(interval / 2) // the slow device's cycle deadline
for {
select {
case r := <-h.cycles:
if r.Device != slow.Slug {
continue // normal devices starting their next cycle
}
if !r.Missed || r.Err == nil {
t.Fatalf("slow device result %+v, want a missed deadline", r)
}
if got := counter("nmon_poll_deadline_missed_total", "device", slow.Slug) - missed0; got != 1 {
t.Fatalf("deadline missed counted %v times", got)
}
return
case <-time.After(10 * time.Second):
t.Fatal("slow device cycle never ended")
}
}
}
func TestICMPStateMachine(t *testing.T) {
clk := clock.NewFake(t0)
st := testutil.OpenStore(t, clk)
d := &store.Device{Name: "vm1", Kind: store.KindVM, Enabled: true, Profile: "generic"}
if err := st.CreateDevice(context.Background(), d); err != nil {
t.Fatal(err)
}
if err := st.SetAddresses(context.Background(), d.ID, []string{"192.0.2.1", "2001:db8::1"}); err != nil {
t.Fatal(err)
}
miss, hit := errors.New("timeout"), error(nil)
h := newHarness(t, clk, st, opts{})
h.ping.results = []error{hit, miss, miss, miss, hit, hit}
h.start(1)
var ups []float64
step := func() {
before := h.ping.count()
for h.ping.count() == before {
clk.Advance(15 * time.Second)
time.Sleep(2 * time.Millisecond)
}
time.Sleep(20 * time.Millisecond) // the write follows the ping
}
for i := 0; i < 6; i++ {
step()
}
for _, s := range h.w.samples("nmon_icmp_up") {
ups = append(ups, s.Value)
}
// hit: up; two misses: nothing changes; third miss: down; first hit: still down; second hit: up.
want := []float64{1, 1, 1, 0, 0, 1}
if len(ups) != len(want) {
t.Fatalf("icmp_up series %v, want %v", ups, want)
}
for i := range want {
if ups[i] != want[i] {
t.Fatalf("icmp_up series %v, want %v", ups, want)
}
}
if n := len(h.w.samples("nmon_rtt_seconds")); n != 3 {
t.Errorf("%d rtt samples, want one per hit", n)
}
for _, s := range h.w.samples("nmon_icmp_up") {
if len(s.Labels) != 1 || s.Labels[0].Value != d.Slug {
t.Fatalf("labels %v", s.Labels)
}
}
}
func TestICMPUnsupportedWritesNothing(t *testing.T) {
clk := clock.NewFake(t0)
st := testutil.OpenStore(t, clk)
fakeDevices(t, st, 1)
h := newHarness(t, clk, st, opts{dial: newBlocker().dial})
h.ping.results = []error{icmp.ErrUnsupported}
h.start(1)
for h.ping.count() == 0 {
clk.Advance(15 * time.Second)
time.Sleep(2 * time.Millisecond)
}
for i := 0; i < 5; i++ {
clk.Advance(15 * time.Second)
time.Sleep(2 * time.Millisecond)
}
if n := h.ping.count(); n != 1 {
t.Errorf("pinged %d times after ErrUnsupported", n)
}
if len(h.w.samples("nmon_icmp_up")) != 0 || len(h.w.samples("nmon_rtt_seconds")) != 0 {
t.Error("unsupported probe wrote samples")
}
}
func TestRefreshPicksUpNewDeviceAndSelfmonIsWritten(t *testing.T) {
clk := clock.NewFake(t0)
st := testutil.OpenStore(t, clk)
b := newBlocker()
close(b.release)
h := newHarness(t, clk, st, opts{dial: b.dial})
h.start(0)
devs := fakeDevices(t, st, 1)
h.p.Refresh()
eventually(t, func() bool { return clk.Waiters() >= 3 })
h.until(func() bool { return h.counts[devs[0].Slug] >= 1 })
eventually(t, func() bool { return len(h.w.samples("nmon_poll_duration_seconds")) > 0 })
if len(h.w.samples("nmon_pdus_total")) == 0 && len(h.w.samples("nmon_poll_duration_seconds")) == 0 {
t.Error("selfmon snapshot not written")
}
}

View File

@@ -0,0 +1,93 @@
// Package testutil builds the fixtures WP5 tests share: a migrated SQLite
// store, simagent devices and inventory rows seeded by discovery.
package testutil
import (
"context"
"path/filepath"
"testing"
"time"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/discovery"
"github.com/v10networks/nmon/internal/secrets"
"github.com/v10networks/nmon/internal/simagent"
"github.com/v10networks/nmon/internal/snmp"
"github.com/v10networks/nmon/internal/store"
)
// OpenStore returns a migrated SQLite store in a temp dir, closed at test end.
func OpenStore(t testing.TB, clk clock.Clock) store.Store {
t.Helper()
dir := t.TempDir()
key := filepath.Join(dir, "secret.key")
if err := secrets.GenerateKeyFile(key); err != nil {
t.Fatal(err)
}
kr, err := secrets.LoadKeyring(key)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
st, err := store.Open(ctx, store.DriverSQLite, filepath.Join(dir, "inventory.db"), kr, clk)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
if err := st.Migrate(ctx); err != nil {
t.Fatal(err)
}
return st
}
// StartAgent serves cfg on [::1] (unless cfg.Addr says otherwise) until the test ends.
func StartAgent(t testing.TB, cfg simagent.Config) *simagent.Agent {
t.Helper()
a, err := simagent.Start(context.Background(), cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { a.Close() })
return a
}
// AddDevice creates a v2c credential with the given community and a network
// device pointing at ::1, and seeds its interfaces from agent a.
func AddDevice(t testing.TB, st store.Store, a *simagent.Agent, name, community string, clk clock.Clock) *store.Device {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
cred := &store.SNMPCredential{Name: name + "-cred", Version: "v2c", Community: community}
if err := st.CreateCredential(ctx, cred); err != nil {
t.Fatal(err)
}
dev := &store.Device{Name: name, Kind: store.KindNetwork, SNMPCredentialID: &cred.ID, Enabled: true, Profile: "generic"}
if err := st.CreateDevice(ctx, dev); err != nil {
t.Fatal(err)
}
if err := st.SetAddresses(ctx, dev.ID, []string{"::1"}); err != nil {
t.Fatal(err)
}
Seed(t, st, a, dev, cred, clk)
got, err := st.GetDeviceByID(ctx, dev.ID)
if err != nil {
t.Fatal(err)
}
return got
}
// Seed runs discovery.Seed for dev against agent a.
func Seed(t testing.TB, st store.Store, a *simagent.Agent, dev *store.Device, cred *store.SNMPCredential, clk clock.Clock) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
tg := discovery.Target(a.Addr().Addr(), int(a.Addr().Port()), cred, clk)
sess, err := snmp.Dial(ctx, tg, "discovery")
if err != nil {
t.Fatal(err)
}
defer sess.Close()
if err := discovery.Seed(ctx, st, sess, dev, clk); err != nil {
t.Fatal(err)
}
}

138
scripts/gate-m1.sh Executable file
View File

@@ -0,0 +1,138 @@
#!/usr/bin/env bash
# Gate M1: `nmon serve` polls the counters fixture and /api/v1/query returns
# rising ifHCInOctets. Procedure: docs/milestones/M1-skeleton-and-fixtures.md.
# Needs: go, make, curl, file, ss, pgrep, python3 (JSON checks).
set -uo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
TMP="$(mktemp -d)"
FAILS=0
NMON_PID="" AGENT_PID=""
pass() { echo "PASS: $*"; }
fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
die() { fail "$*"; exit 1; }
cleanup() {
local pid
for pid in $NMON_PID $AGENT_PID; do
kill "$pid" 2>/dev/null || true
done
for pid in $NMON_PID $AGENT_PID; do
for _ in $(seq 1 100); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done
kill -9 "$pid" 2>/dev/null || true
done
if [ "$FAILS" -ne 0 ] && [ -f "$TMP/nmon.log" ]; then
echo "--- tail of nmon.log"; tail -n 40 "$TMP/nmon.log"
fi
rm -rf "$TMP"
}
trap cleanup EXIT
trap 'exit 130' INT TERM
pick_port() {
local p
while :; do
p=$((20000 + RANDOM % 30000))
ss -H -lntu 2>/dev/null | awk '{print $5}' | grep -q "[:.]$p\$" || { echo "$p"; return; }
done
}
command -v python3 >/dev/null || die "python3 is required for the JSON checks"
# 1. Build with the embedded stores; the binary must be static.
echo "== build"
make build >"$TMP/build.log" 2>&1 || { tail -n 20 "$TMP/build.log"; die "make build"; }
if file nmon | grep -q 'statically linked'; then pass "nmon is statically linked"; else die "nmon is not statically linked: $(file nmon)"; fi
# 2. Fixture agent on [::1]:<port>.
echo "== simagent"
SNMP_PORT="$(pick_port)"
go build -o "$TMP/simagent" ./internal/simagent/cmd/simagent || die "build simagent"
"$TMP/simagent" -addr "[::1]:$SNMP_PORT" -rec testdata/snmprec/counters.snmprec >"$TMP/simagent.log" 2>&1 &
AGENT_PID=$!
for _ in $(seq 1 50); do grep -q listening "$TMP/simagent.log" 2>/dev/null && break; sleep 0.1; done
grep -q listening "$TMP/simagent.log" || { cat "$TMP/simagent.log"; die "simagent did not start"; }
pass "simagent serving counters.snmprec on [::1]:$SNMP_PORT"
# 3. nmon serve, data dir and key file in separate places.
echo "== serve"
HTTP_PORT="$(pick_port)"
BASE="http://[::1]:$HTTP_PORT"
./nmon serve --data-dir "$TMP/data" --secret-key-file "$TMP/keys/secret.key" --insecure-http \
--listen "[::1]:$HTTP_PORT" --poll-interval 10s --snmp-port "$SNMP_PORT" >"$TMP/nmon.log" 2>&1 &
NMON_PID=$!
for _ in $(seq 1 300); do
curl -fsS "$BASE/openapi.json" >/dev/null 2>&1 && break
kill -0 "$NMON_PID" 2>/dev/null || break
sleep 0.2
done
curl -fsS "$BASE/openapi.json" >/dev/null 2>&1 || die "nmon serve did not come up"
pass "nmon serve is up on $BASE"
# 4. Add the fixture device.
echo "== add device"
CODE="$(curl -sS -o "$TMP/device.json" -w '%{http_code}' -X POST "$BASE/api/v1/devices" \
-H 'Content-Type: application/json' \
-d '{"name":"counters","addresses":["::1"],"snmp":{"version":"v2c","community":"public"}}')"
[ "$CODE" = 201 ] || { cat "$TMP/device.json"; die "POST /api/v1/devices returned $CODE"; }
SLUG="$(python3 -I -c 'import json,sys; print(json.load(open(sys.argv[1]))["slug"])' "$TMP/device.json")"
NIF="$(python3 -I -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["interfaces"]))' "$TMP/device.json")"
[ "$NIF" -ge 1 ] || die "device $SLUG has no interfaces"
pass "device $SLUG added with $NIF interfaces"
# 5. Three cycles, then query twice one cycle apart.
echo "== query (waiting for 3 poll cycles)"
sleep 30
query() { curl -fsS --get "$BASE/api/v1/query" --data-urlencode 'q=ifHCInOctets' -o "$1"; }
for _ in $(seq 1 15); do
query "$TMP/q1.json" && python3 -I -c 'import json,sys; sys.exit(0 if json.load(open(sys.argv[1]))["data"]["result"] else 1)' "$TMP/q1.json" && break
sleep 2
done
sleep 10
CHECK='
import json, sys
a = json.load(open(sys.argv[1]))["data"]["result"]
b = {s["metric"].get("iface"): float(s["value"][1]) for s in json.load(open(sys.argv[2]))["data"]["result"]}
if not a:
print("no ifHCInOctets series"); sys.exit(1)
for s in a:
keys = sorted(s["metric"])
if keys != ["__name__", "device", "iface"]:
print("labels are", keys); sys.exit(1)
i = s["metric"]["iface"]
if i not in b or not b[i] > float(s["value"][1]):
print("iface", i, "did not rise:", s["value"][1], "->", b.get(i)); sys.exit(1)
print(len(a), "series, labels exactly {__name__, device, iface}, all values strictly rising")
'
OK=0
for _ in $(seq 1 6); do
query "$TMP/q2.json" && OUT="$(python3 -I -c "$CHECK" "$TMP/q1.json" "$TMP/q2.json" 2>&1)" && { OK=1; break; }
sleep 2
done
if [ "$OK" = 1 ]; then pass "ifHCInOctets: $OUT"; else fail "ifHCInOctets check: ${OUT:-query failed}"; fi
# 6. The metric stores listen on loopback only.
echo "== listeners"
PIDS="$(pgrep -P "$NMON_PID" | tr '\n' ' ')"
N=0 BAD=0
while read -r addr pid; do
N=$((N + 1))
case "$addr" in 127.0.0.1:*) ;; *) BAD=$((BAD + 1)); echo " store pid $pid listens on $addr" ;; esac
done < <(ss -H -ltnp | awk '{
n = split($0, parts, "pid="); for (i = 2; i <= n; i++) { split(parts[i], p, ","); print $4, p[1] } }' |
while read -r addr pid; do for c in $PIDS; do [ "$c" = "$pid" ] && echo "$addr $pid"; done; done)
if [ "$N" -ge 2 ] && [ "$BAD" -eq 0 ]; then pass "$N store listeners, all on 127.0.0.1"
elif [ "$N" -lt 2 ]; then fail "found $N store listeners (children: ${PIDS:-none}), expected hot and long"
else fail "$BAD store listeners are not on 127.0.0.1"; fi
# 7. Graceful shutdown.
echo "== shutdown"
kill -TERM "$NMON_PID"
if wait "$NMON_PID"; then pass "nmon exited cleanly on SIGTERM"; else fail "nmon exited with status $?"; fi
NMON_PID=""
if [ "$FAILS" -eq 0 ]; then echo "GATE M1: PASS"; else echo "GATE M1: FAIL ($FAILS)"; exit 1; fi