M1 WP5: seed discovery, poller, API, serve wiring, gate script; M1 gate passed
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
This commit is contained in:
2
Makefile
2
Makefile
@@ -43,7 +43,7 @@ test-pg:
|
||||
test-v6only:
|
||||
NMON_TEST_V6ONLY=1 $(GO) test -race -tags 'integration $(TAGS)' ./...
|
||||
|
||||
gate-m%:
|
||||
gate-m1 gate-m2 gate-m3 gate-m4 gate-m5 gate-m6: gate-m%:
|
||||
./scripts/gate-m$*.sh
|
||||
|
||||
clean:
|
||||
|
||||
60
cmd/nmon/migrate.go
Normal file
60
cmd/nmon/migrate.go
Normal file
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// migrate applies pending inventory migrations (up, the default) or reverts the
|
||||
// latest group (down). Credentials are not touched, so no key file is needed.
|
||||
func migrate(ctx context.Context, args []string) error {
|
||||
action := "up"
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
action, args = args[0], args[1:]
|
||||
}
|
||||
if action != "up" && action != "down" {
|
||||
return fmt.Errorf("unknown action %q: want up or down", action)
|
||||
}
|
||||
var f config.Flags
|
||||
fs := newFlagSet("migrate", "migrate [up|down] --data-dir DIR [flags]")
|
||||
storeFlags(fs, &f)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return flagErr(err)
|
||||
}
|
||||
if f.DataDir == "" {
|
||||
return fmt.Errorf("--data-dir is required")
|
||||
}
|
||||
if err := os.MkdirAll(f.DataDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
dsn := f.DBDSN
|
||||
if dsn == "" {
|
||||
if f.DBDriver != store.DriverSQLite {
|
||||
return fmt.Errorf("--db-dsn is required for --db-driver %s", f.DBDriver)
|
||||
}
|
||||
dsn = config.InventoryDB(f.DataDir)
|
||||
}
|
||||
st, err := store.Open(ctx, f.DBDriver, dsn, nil, clock.Real())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer st.Close()
|
||||
if action == "down" {
|
||||
if err := st.Rollback(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("reverted the latest migration group")
|
||||
return nil
|
||||
}
|
||||
if err := st.Migrate(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("migrations up to date")
|
||||
return nil
|
||||
}
|
||||
77
cmd/nmon/serve.go
Normal file
77
cmd/nmon/serve.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
|
||||
"github.com/v10networks/nmon/internal/app"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
)
|
||||
|
||||
func init() {
|
||||
register("serve", "run the monitor: collect, store and serve the API", serve)
|
||||
register("migrate", "apply (up) or revert (down) inventory migrations", migrate)
|
||||
}
|
||||
|
||||
// hidden flags are dev/test only and left out of the usage text.
|
||||
var hidden = map[string]bool{"poll-interval": true, "snmp-port": true}
|
||||
|
||||
// newFlagSet returns a FlagSet whose usage skips hidden flags.
|
||||
func newFlagSet(name, usage string) *flag.FlagSet {
|
||||
fs := flag.NewFlagSet(name, flag.ContinueOnError)
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintf(fs.Output(), "usage: nmon %s\n\nflags:\n", usage)
|
||||
fs.VisitAll(func(f *flag.Flag) {
|
||||
if hidden[f.Name] {
|
||||
return
|
||||
}
|
||||
def := ""
|
||||
if f.DefValue != "" && f.DefValue != "false" && f.DefValue != "0" {
|
||||
def = fmt.Sprintf(" (default %s)", f.DefValue)
|
||||
}
|
||||
fmt.Fprintf(fs.Output(), " --%s\n %s%s\n", f.Name, f.Usage, def)
|
||||
})
|
||||
}
|
||||
return fs
|
||||
}
|
||||
|
||||
// storeFlags registers the flags every inventory-using command shares.
|
||||
func storeFlags(fs *flag.FlagSet, f *config.Flags) {
|
||||
fs.StringVar(&f.DataDir, "data-dir", "", "data folder: inventory.db, vm-hot/, vm-long/, bin/, profiles/, config.yaml (required)")
|
||||
fs.StringVar(&f.DBDriver, "db-driver", "sqlite", "inventory database: sqlite, postgres or mysql")
|
||||
fs.StringVar(&f.DBDSN, "db-dsn", "", "database DSN; default <data-dir>/inventory.db for sqlite")
|
||||
}
|
||||
|
||||
func serve(ctx context.Context, args []string) error {
|
||||
var f config.Flags
|
||||
fs := newFlagSet("serve", "serve --data-dir DIR --insecure-http [flags]")
|
||||
storeFlags(fs, &f)
|
||||
fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile, "SNMP credential key file; created on first run; must be outside --data-dir")
|
||||
fs.StringVar(&f.Listen, "listen", app.DefaultListen, "address for the API")
|
||||
fs.BoolVar(&f.InsecureHTTP, "insecure-http", false, "serve plain HTTP with no authentication (required until TLS and auth arrive in M6)")
|
||||
fs.StringVar(&f.VMHotURL, "vm-hot-url", "", "use this VictoriaMetrics instance as the hot store instead of the embedded one")
|
||||
fs.StringVar(&f.VMLongURL, "vm-long-url", "", "use this VictoriaMetrics instance as the long store instead of the embedded one")
|
||||
fs.IntVar(&f.MaxInflight, "max-inflight", 512, "most SNMP polls in flight at once")
|
||||
fs.DurationVar(&f.PollInterval, "poll-interval", 0, "dev: SNMP poll interval (default 60s)")
|
||||
fs.IntVar(&f.SNMPPort, "snmp-port", 0, "dev: UDP port of every device's agent (default 161)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return flagErr(err)
|
||||
}
|
||||
if f.DataDir == "" {
|
||||
return fmt.Errorf("--data-dir is required")
|
||||
}
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})))
|
||||
slog.Info("nmon starting", "data_dir", f.DataDir, "listen", f.Listen)
|
||||
return app.Run(ctx, f)
|
||||
}
|
||||
|
||||
// flagErr turns flag.ErrHelp into a clean exit.
|
||||
func flagErr(err error) error {
|
||||
if err == flag.ErrHelp {
|
||||
os.Exit(0)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -90,4 +90,22 @@ WP2, WP3 and WP4 run in parallel. WP5 runs after them.
|
||||
|
||||
## Gate log
|
||||
|
||||
_(filled in by the supervisor when the gate passes)_
|
||||
**Passed 2026-10-22**, run by the supervisor (Opus) on linux/amd64 (16 cores, 15 GB), Go 1.27.1, VictoriaMetrics v1.153.0.
|
||||
|
||||
| Check | Result |
|
||||
|---|---|
|
||||
| `gofmt -l`, `go vet ./...` | clean |
|
||||
| `go test -race ./...` | 13 packages ok |
|
||||
| `go test -race -tags 'integration vmembed' ./...` | all ok, including real-VM supervisor tests (kill -9 → healthy in about 1.4 s, listeners on 127.0.0.1 only) and in-process `app.Run` |
|
||||
| `./scripts/test-pg.sh` (Postgres 16) | store suite ok on SQLite and Postgres; MySQL 8 suite also run once by WP2 |
|
||||
| `make gate-m1` | `GATE M1: PASS`: static binary; counters fixture on `[::1]`; 8 `ifHCInOctets` series with labels exactly `{__name__, device, iface}`, all strictly rising; 2 store listeners, both on 127.0.0.1; clean SIGTERM exit |
|
||||
|
||||
**Decisions taken in M1** (beyond DEVIATIONS 1–5):
|
||||
- `POST /api/v1/devices` walks the device before writing. An unreachable device returns 502 and nothing is stored, so no slug is burned.
|
||||
- The 100k-series query cap is enforced on the result (422), not with a store flag, because rollups need more headroom.
|
||||
- Supervisor: 3 restarts in 10 min are allowed; the 4th failure raises `store_failed` and stops the loop. The children run with `-search.latencyOffset=0s`.
|
||||
- The init migration uses frozen struct copies. Credential ciphertext is bound to its row (AAD includes the row id).
|
||||
- The dev-only flags `--poll-interval` and `--snmp-port` exist for gates and tests.
|
||||
|
||||
**Carried into M2:** a per-device SNMP port (a `devices.snmp_port` column replacing the global `--snmp-port`, which the loadtest needs); varbind adaptation, backoff and degraded state; the 30 s poll for 32-bit fast links; periodic re-discovery; full identity matching (`TODO(M2)` in `internal/discovery`).
|
||||
**Carried into M3:** `/docs` currently loads Swagger UI assets from a CDN; self-host them with the embedded UI.
|
||||
|
||||
@@ -11,8 +11,8 @@ its **Gate log** is filled in when the gate passes. A milestone is done only whe
|
||||
|
||||
| # | Milestone | Gate | Doc | Status |
|
||||
|---|---|---|---|---|
|
||||
| M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | in progress |
|
||||
| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | planned |
|
||||
| M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | **passed**, tagged `m1` |
|
||||
| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | in progress (WP1 done) |
|
||||
| M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | planned |
|
||||
| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | planned |
|
||||
| M5 | Push ingest, rollup, checks | a replayed Proxmox sample shows under its node and a 2 y query hits the 1 h tier | [M5](M5-ingest-rollup-checks.md) | planned |
|
||||
|
||||
1
go.mod
1
go.mod
@@ -3,6 +3,7 @@ module github.com/v10networks/nmon
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
github.com/danielgtaylor/huma/v2 v2.39.1
|
||||
github.com/go-sql-driver/mysql v1.10.1
|
||||
github.com/gosnmp/gosnmp v1.46.0
|
||||
github.com/oklog/ulid/v2 v2.1.2
|
||||
|
||||
8
go.sum
8
go.sum
@@ -2,8 +2,14 @@ filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/danielgtaylor/huma/v2 v2.39.1 h1:0kwF4ltQoYZ+IU55VPy+BcGekzgF44R64daTGde1H+g=
|
||||
github.com/danielgtaylor/huma/v2 v2.39.1/go.mod h1:zcnQ38duIJ3VUHwFaBoZ6x8T+KN/mr33oyqxcj0HTug=
|
||||
github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
|
||||
github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
|
||||
github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78=
|
||||
github.com/fxamacker/cbor/v2 v2.9.2/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
||||
github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
|
||||
github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
||||
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
@@ -51,6 +57,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
|
||||
154
internal/api/api.go
Normal file
154
internal/api/api.go
Normal file
@@ -0,0 +1,154 @@
|
||||
// Package api is nmon's HTTP API: huma v2 on net/http, errors as RFC 9457
|
||||
// problem details, OpenAPI generated from the handlers at /openapi.json.
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
"github.com/danielgtaylor/huma/v2/adapters/humago"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/selfmon"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// Limits.
|
||||
const (
|
||||
// QueryTimeout bounds one /query call.
|
||||
QueryTimeout = 30 * time.Second
|
||||
// MaxSeries is the most series a /query call may return; more is a 422.
|
||||
// It is enforced on the result rather than with a store flag because the
|
||||
// rollup job needs the store's own limit far higher (-search.maxUniqueTimeseries).
|
||||
MaxSeries = 100_000
|
||||
// DiscoveryTimeout bounds the seed discovery run by POST /devices.
|
||||
DiscoveryTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// Querier runs read-only MetricsQL against the hot store. *vmsup.Querier implements it.
|
||||
type Querier interface {
|
||||
Query(ctx context.Context, store, q string, ts time.Time) ([]byte, error)
|
||||
QueryRange(ctx context.Context, store, q string, from, to time.Time, step time.Duration) ([]byte, error)
|
||||
}
|
||||
|
||||
// Refresher is told when the device list changed. *poller.Poller implements it.
|
||||
type Refresher interface{ Refresh() }
|
||||
|
||||
// Deps are what the handlers need. Store, Querier and Poller are required.
|
||||
type Deps struct {
|
||||
Store store.Store
|
||||
Querier Querier
|
||||
Poller Refresher
|
||||
Clock clock.Clock // default clock.Real()
|
||||
Registry *selfmon.Registry // served at /metrics; default selfmon.Default
|
||||
Logger *slog.Logger // default slog.Default()
|
||||
SNMPPort int // dev/test override of UDP 161; 0 = 161
|
||||
|
||||
// DiscoveryTimeout overrides the 30 s seed discovery limit (tests).
|
||||
DiscoveryTimeout time.Duration
|
||||
}
|
||||
|
||||
type server struct {
|
||||
Deps
|
||||
}
|
||||
|
||||
// New returns the API handler with every route registered.
|
||||
func New(d Deps) http.Handler {
|
||||
if d.Clock == nil {
|
||||
d.Clock = clock.Real()
|
||||
}
|
||||
if d.Registry == nil {
|
||||
d.Registry = selfmon.Default
|
||||
}
|
||||
if d.Logger == nil {
|
||||
d.Logger = slog.Default()
|
||||
}
|
||||
if d.DiscoveryTimeout <= 0 {
|
||||
d.DiscoveryTimeout = DiscoveryTimeout
|
||||
}
|
||||
s := &server{d}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
cfg := huma.DefaultConfig("nmon", "1.0.0")
|
||||
cfg.Info.Description = "nmon network monitor API."
|
||||
cfg.OpenAPIPath = "/openapi"
|
||||
cfg.DocsPath = "/docs"
|
||||
cfg.DocsRenderer = huma.DocsRendererSwaggerUI
|
||||
cfg.CreateHooks = nil // no $schema fields or Link headers in responses
|
||||
api := humago.New(mux, cfg)
|
||||
|
||||
s.registerDevices(api)
|
||||
s.registerQuery(api)
|
||||
s.registerMetrics(api)
|
||||
return problemFallback(mux)
|
||||
}
|
||||
|
||||
// storeErr maps store sentinels to problem responses; anything else is a 500
|
||||
// whose detail does not leak internals.
|
||||
func (s *server) storeErr(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, store.ErrNotFound):
|
||||
return huma.Error404NotFound("not found")
|
||||
case errors.Is(err, store.ErrConflict):
|
||||
return huma.Error409Conflict(trimSentinel(err, store.ErrConflict))
|
||||
case errors.Is(err, store.ErrInvalid):
|
||||
return huma.Error422UnprocessableEntity(trimSentinel(err, store.ErrInvalid))
|
||||
}
|
||||
s.Logger.Error("store error", "err", err)
|
||||
return huma.Error500InternalServerError("internal error")
|
||||
}
|
||||
|
||||
// trimSentinel returns err's text without the leading "store: x: " sentinel.
|
||||
func trimSentinel(err, sentinel error) string {
|
||||
msg := err.Error()
|
||||
if rest, ok := strings.CutPrefix(msg, sentinel.Error()+": "); ok {
|
||||
return rest
|
||||
}
|
||||
return strings.TrimPrefix(msg, "store: ")
|
||||
}
|
||||
|
||||
// problemFallback answers requests that match no route (404, 405) with
|
||||
// problem+json instead of the mux's plain text.
|
||||
func problemFallback(mux *http.ServeMux) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h, pattern := mux.Handler(r)
|
||||
if pattern != "" {
|
||||
mux.ServeHTTP(w, r) // sets the path values
|
||||
return
|
||||
}
|
||||
cw := &capture{h: http.Header{}}
|
||||
h.ServeHTTP(cw, r)
|
||||
status := cw.status
|
||||
if status == 0 {
|
||||
status = http.StatusNotFound
|
||||
}
|
||||
if allow := cw.h.Get("Allow"); allow != "" {
|
||||
w.Header().Set("Allow", allow)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/problem+json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"title": http.StatusText(status), "status": status,
|
||||
"detail": r.Method + " " + r.URL.Path + ": " + strings.ToLower(http.StatusText(status)),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// capture records only the status and headers the mux would have written.
|
||||
type capture struct {
|
||||
h http.Header
|
||||
status int
|
||||
}
|
||||
|
||||
func (c *capture) Header() http.Header { return c.h }
|
||||
func (c *capture) Write(b []byte) (int, error) { return len(b), nil }
|
||||
func (c *capture) WriteHeader(s int) { c.status = s }
|
||||
|
||||
func itoa(n int) string { return strconv.Itoa(n) }
|
||||
272
internal/api/api_test.go
Normal file
272
internal/api/api_test.go
Normal file
@@ -0,0 +1,272 @@
|
||||
package api_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/fixturegen"
|
||||
"github.com/v10networks/nmon/internal/selfmon"
|
||||
"github.com/v10networks/nmon/internal/simagent"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
"github.com/v10networks/nmon/internal/testutil"
|
||||
"github.com/v10networks/nmon/internal/ulid"
|
||||
"github.com/v10networks/nmon/internal/vmsup"
|
||||
)
|
||||
|
||||
type querier struct {
|
||||
body []byte
|
||||
err error
|
||||
q string
|
||||
ts, from, to time.Time
|
||||
step time.Duration
|
||||
instant, rangeCnt int
|
||||
}
|
||||
|
||||
func (q *querier) Query(_ context.Context, st, expr string, ts time.Time) ([]byte, error) {
|
||||
q.instant++
|
||||
q.q, q.ts = expr, ts
|
||||
return q.body, q.err
|
||||
}
|
||||
|
||||
func (q *querier) QueryRange(_ context.Context, st, expr string, from, to time.Time, step time.Duration) ([]byte, error) {
|
||||
q.rangeCnt++
|
||||
q.q, q.from, q.to, q.step = expr, from, to, step
|
||||
return q.body, q.err
|
||||
}
|
||||
|
||||
type refresher struct{ n atomic.Int32 }
|
||||
|
||||
func (r *refresher) Refresh() { r.n.Add(1) }
|
||||
|
||||
type env struct {
|
||||
h http.Handler
|
||||
st store.Store
|
||||
q *querier
|
||||
ref *refresher
|
||||
reg *selfmon.Registry
|
||||
}
|
||||
|
||||
func newEnv(t *testing.T, port int) *env {
|
||||
clk := clock.NewFake(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC))
|
||||
e := &env{st: testutil.OpenStore(t, clk), q: &querier{body: []byte(`{"status":"success","data":{"resultType":"vector","result":[]}}`)},
|
||||
ref: &refresher{}, reg: selfmon.New()}
|
||||
e.h = api.New(api.Deps{Store: e.st, Querier: e.q, Poller: e.ref, Clock: clk, Registry: e.reg, SNMPPort: port,
|
||||
DiscoveryTimeout: 3 * time.Second})
|
||||
return e
|
||||
}
|
||||
|
||||
func (e *env) do(method, path, body string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(method, path, strings.NewReader(body))
|
||||
if body != "" {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
e.h.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func isProblem(t *testing.T, rec *httptest.ResponseRecorder, status int) {
|
||||
t.Helper()
|
||||
if rec.Code != status {
|
||||
t.Fatalf("status %d, want %d: %s", rec.Code, status, rec.Body)
|
||||
}
|
||||
if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/problem+json") {
|
||||
t.Fatalf("content type %q, want problem+json: %s", ct, rec.Body)
|
||||
}
|
||||
var p struct {
|
||||
Title string `json:"title"`
|
||||
Status int `json:"status"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &p); err != nil || p.Status != status || p.Title == "" {
|
||||
t.Fatalf("problem body %s (%v)", rec.Body, err)
|
||||
}
|
||||
}
|
||||
|
||||
const community = "s3cr3t-comm-xyz"
|
||||
|
||||
func TestAddAndGetDevice(t *testing.T) {
|
||||
a := testutil.StartAgent(t, simagent.Config{
|
||||
Recording: fixturegen.Generic(fixturegen.Options{Ports: 8, SysName: "edge1", HCCounters: true}), Community: community,
|
||||
})
|
||||
e := newEnv(t, int(a.Addr().Port()))
|
||||
body := fmt.Sprintf(`{"name":"Edge One","addresses":["::1"],"site":"yyz","role":"edge","snmp":{"version":"v2c","community":%q}}`, community)
|
||||
rec := e.do("POST", "/api/v1/devices", body)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("POST: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
if e.ref.n.Load() != 1 {
|
||||
t.Error("poller not refreshed")
|
||||
}
|
||||
var created api.DeviceView
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &created); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if created.Slug == "" || created.Name != "Edge One" || created.Site != "yyz" || created.SysName != "edge1" ||
|
||||
created.MgmtIP != "::1" || created.SNMPVersion != "v2c" || len(created.Interfaces) != 8 {
|
||||
t.Fatalf("created: %+v", created)
|
||||
}
|
||||
|
||||
get := e.do("GET", "/api/v1/devices/"+created.Slug, "")
|
||||
if get.Code != 200 {
|
||||
t.Fatalf("GET: %d %s", get.Code, get.Body)
|
||||
}
|
||||
for _, r := range []*httptest.ResponseRecorder{rec, get} {
|
||||
if strings.Contains(r.Body.String(), community) {
|
||||
t.Fatalf("response contains the community: %s", r.Body)
|
||||
}
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal(r.Body.Bytes(), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{"id", "snmp_credential_id", "community", "device_id"} {
|
||||
if _, ok := m[k]; ok {
|
||||
t.Errorf("device response has key %q", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
var got api.DeviceView
|
||||
if err := json.Unmarshal(get.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, i := range got.Interfaces {
|
||||
if !ulid.Valid(i.ID) {
|
||||
t.Errorf("interface id %q", i.ID)
|
||||
}
|
||||
if i.FirstSeen.Location() != time.UTC {
|
||||
t.Errorf("timestamp not UTC: %v", i.FirstSeen)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(get.Body.String(), `"created_at":"2026-10-22T12:00:00Z"`) {
|
||||
t.Errorf("created_at is not RFC 3339 UTC: %s", get.Body)
|
||||
}
|
||||
// the secret is in the store, encrypted, and decrypts for the poller
|
||||
d, _ := e.st.GetDeviceBySlug(context.Background(), created.Slug)
|
||||
c, err := e.st.GetCredential(context.Background(), *d.SNMPCredentialID)
|
||||
if err != nil || c.Community != community {
|
||||
t.Fatalf("stored credential: %+v %v", c, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddDeviceRejections(t *testing.T) {
|
||||
e := newEnv(t, 1) // nothing listens on UDP 1
|
||||
for name, tc := range map[string]struct {
|
||||
body string
|
||||
status int
|
||||
}{
|
||||
"no snmp for network": {`{"name":"x","addresses":["::1"]}`, 422},
|
||||
"bad address": {`{"name":"x","addresses":["nope"],"snmp":{"version":"v2c","community":"c"}}`, 422},
|
||||
"no community": {`{"name":"x","addresses":["::1"],"snmp":{"version":"v2c"}}`, 422},
|
||||
"bad version": {`{"name":"x","addresses":["::1"],"snmp":{"version":"v9","community":"c"}}`, 422},
|
||||
"no addresses": {`{"name":"x","addresses":[],"snmp":{"version":"v2c","community":"c"}}`, 422},
|
||||
"unreachable": {`{"name":"x","addresses":["::1"],"snmp":{"version":"v2c","community":"c"}}`, 502},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) { isProblem(t, e.do("POST", "/api/v1/devices", tc.body), tc.status) })
|
||||
}
|
||||
if _, n, err := e.st.ListDevices(context.Background(), store.DeviceFilter{}, store.Page{}); err != nil || n != 0 {
|
||||
t.Fatalf("rejected requests stored %d devices (%v)", n, err)
|
||||
}
|
||||
if e.ref.n.Load() != 0 {
|
||||
t.Error("poller refreshed for a rejected device")
|
||||
}
|
||||
// a device without SNMP is fine
|
||||
rec := e.do("POST", "/api/v1/devices", `{"name":"vm1","kind":"vm","addresses":["192.0.2.7"]}`)
|
||||
if rec.Code != 201 {
|
||||
t.Fatalf("vm: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProblemDetails(t *testing.T) {
|
||||
e := newEnv(t, 0)
|
||||
isProblem(t, e.do("GET", "/api/v1/devices/no-such-device", ""), 404)
|
||||
isProblem(t, e.do("GET", "/api/v1/nothing", ""), 404)
|
||||
rec := e.do("DELETE", "/api/v1/query", "")
|
||||
isProblem(t, rec, 405)
|
||||
isProblem(t, e.do("GET", "/api/v1/query", ""), 422) // missing q
|
||||
}
|
||||
|
||||
func TestOpenAPIDocsAndMetrics(t *testing.T) {
|
||||
e := newEnv(t, 0)
|
||||
e.reg.Counter("nmon_test_total", "device", "d1").Add(3)
|
||||
rec := e.do("GET", "/openapi.json", "")
|
||||
var spec struct {
|
||||
Paths map[string]map[string]any `json:"paths"`
|
||||
}
|
||||
if rec.Code != 200 || json.Unmarshal(rec.Body.Bytes(), &spec) != nil {
|
||||
t.Fatalf("openapi.json: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
for _, p := range []string{"/api/v1/devices", "/api/v1/devices/{slug}", "/api/v1/query", "/api/v1/query_range", "/metrics"} {
|
||||
if spec.Paths[p] == nil {
|
||||
t.Errorf("openapi.json lacks %s", p)
|
||||
}
|
||||
}
|
||||
if d := e.do("GET", "/docs", ""); d.Code != 200 || !strings.Contains(d.Header().Get("Content-Type"), "text/html") {
|
||||
t.Errorf("/docs: %d %s", d.Code, d.Header().Get("Content-Type"))
|
||||
}
|
||||
m := e.do("GET", "/metrics", "")
|
||||
if m.Code != 200 || !strings.Contains(m.Body.String(), `nmon_test_total{device="d1"} 3`) {
|
||||
t.Errorf("/metrics: %d %s", m.Code, m.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func seriesBody(n int) []byte {
|
||||
var b bytes.Buffer
|
||||
b.WriteString(`{"status":"success","data":{"resultType":"vector","result":[`)
|
||||
for i := 0; i < n; i++ {
|
||||
if i > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(`{"metric":{},"value":[1,"1"]}`)
|
||||
}
|
||||
b.WriteString(`]}}`)
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func TestQueryProxy(t *testing.T) {
|
||||
e := newEnv(t, 0)
|
||||
e.q.body = seriesBody(3)
|
||||
rec := e.do("GET", "/api/v1/query?q=ifHCInOctets&time=2026-10-22T12:00:00Z", "")
|
||||
if rec.Code != 200 || !bytes.Equal(rec.Body.Bytes(), e.q.body) || rec.Header().Get("Content-Type") != "application/json" {
|
||||
t.Fatalf("passthrough: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
if e.q.q != "ifHCInOctets" || !e.q.ts.Equal(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)) {
|
||||
t.Errorf("forwarded %q at %v", e.q.q, e.q.ts)
|
||||
}
|
||||
e.do("GET", "/api/v1/query?q=up&time=1700000000.5", "")
|
||||
if e.q.ts.UnixMilli() != 1700000000500 {
|
||||
t.Errorf("unix time parsed as %v", e.q.ts)
|
||||
}
|
||||
isProblem(t, e.do("GET", "/api/v1/query?q=up&time=yesterday", ""), 422)
|
||||
|
||||
rec = e.do("GET", "/api/v1/query_range?q=up&from=1700000000&to=1700003600&step=30s", "")
|
||||
if rec.Code != 200 || e.q.step != 30*time.Second || e.q.to.Sub(e.q.from) != time.Hour {
|
||||
t.Fatalf("range: %d %s step=%v", rec.Code, rec.Body, e.q.step)
|
||||
}
|
||||
isProblem(t, e.do("GET", "/api/v1/query_range?q=up", ""), 422) // step required
|
||||
isProblem(t, e.do("GET", "/api/v1/query_range?q=up&step=0", ""), 422)
|
||||
|
||||
e.q.body = seriesBody(api.MaxSeries)
|
||||
if rec := e.do("GET", "/api/v1/query?q=x", ""); rec.Code != 200 {
|
||||
t.Fatalf("exactly %d series must pass: %d", api.MaxSeries, rec.Code)
|
||||
}
|
||||
e.q.body = seriesBody(api.MaxSeries + 1)
|
||||
isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 422)
|
||||
isProblem(t, e.do("GET", "/api/v1/query_range?q=x&step=1m", ""), 422)
|
||||
|
||||
e.q.err = fmt.Errorf("%w: hot", vmsup.ErrStoreDown)
|
||||
isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 503)
|
||||
e.q.err = errors.New("vmsup: hot store: 422 Unprocessable Entity: cannot parse")
|
||||
isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 422)
|
||||
e.q.err = errors.New("connection reset")
|
||||
isProblem(t, e.do("GET", "/api/v1/query?q=x", ""), 502)
|
||||
}
|
||||
281
internal/api/devices.go
Normal file
281
internal/api/devices.go
Normal file
@@ -0,0 +1,281 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"time"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/discovery"
|
||||
"github.com/v10networks/nmon/internal/snmp"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
"github.com/v10networks/nmon/internal/ulid"
|
||||
)
|
||||
|
||||
// SNMPBody is the SNMP credential in a request. Secrets are write-only: no
|
||||
// response ever contains them.
|
||||
type SNMPBody struct {
|
||||
Version string `json:"version" enum:"v1,v2c,v3" doc:"SNMP version."`
|
||||
Community string `json:"community,omitempty" doc:"v1 and v2c community." example:"public"`
|
||||
V3User string `json:"v3_user,omitempty" doc:"v3 user name."`
|
||||
V3AuthProto string `json:"v3_auth_proto,omitempty" doc:"MD5, SHA, SHA224, SHA256 (default), SHA384 or SHA512."`
|
||||
V3AuthPass string `json:"v3_auth_pass,omitempty"`
|
||||
V3PrivProto string `json:"v3_priv_proto,omitempty" doc:"DES, AES (default), AES192 or AES256."`
|
||||
V3PrivPass string `json:"v3_priv_pass,omitempty"`
|
||||
V3Context string `json:"v3_context,omitempty"`
|
||||
}
|
||||
|
||||
// AddDeviceBody is the body of POST /api/v1/devices.
|
||||
type AddDeviceBody struct {
|
||||
Name string `json:"name" minLength:"1" maxLength:"255" example:"core-sw1"`
|
||||
Addresses []string `json:"addresses" minItems:"1" doc:"Management addresses; an IPv6 address is preferred over IPv4." example:"[\"2001:db8::1\"]"`
|
||||
Kind string `json:"kind,omitempty" enum:"network,vm,container,check_only" default:"network"`
|
||||
Site string `json:"site,omitempty"`
|
||||
Role string `json:"role,omitempty"`
|
||||
SNMP *SNMPBody `json:"snmp,omitempty" doc:"Required for kind network."`
|
||||
}
|
||||
|
||||
// InterfaceView is an interface as the API shows it, addressed by ULID.
|
||||
type InterfaceView struct {
|
||||
ID string `json:"id" doc:"Immutable ULID."`
|
||||
IfIndex int64 `json:"ifindex" doc:"Current ifIndex; may change, the ID does not."`
|
||||
IfName string `json:"ifname"`
|
||||
IfDescr string `json:"ifdescr"`
|
||||
IfAlias string `json:"ifalias"`
|
||||
IfPhysAddr string `json:"ifphysaddr"`
|
||||
IfSpeed int64 `json:"ifspeed" doc:"bits per second"`
|
||||
IfHighSpeed int64 `json:"ifhighspeed" doc:"megabits per second"`
|
||||
IfType int64 `json:"iftype"`
|
||||
AdminStatus int64 `json:"admin_status"`
|
||||
OperStatus int64 `json:"oper_status"`
|
||||
HasHCCounters bool `json:"has_hc_counters"`
|
||||
Excluded bool `json:"excluded"`
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastDiscovered time.Time `json:"last_discovered"`
|
||||
}
|
||||
|
||||
// DeviceView is a device as the API shows it. It carries no numeric ID and no credential.
|
||||
type DeviceView struct {
|
||||
Slug string `json:"slug" doc:"Immutable public key."`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Site string `json:"site"`
|
||||
Role string `json:"role"`
|
||||
Profile string `json:"profile"`
|
||||
Enabled bool `json:"enabled"`
|
||||
MgmtIP string `json:"mgmt_ip"`
|
||||
Addresses []string `json:"addresses"`
|
||||
SNMPVersion string `json:"snmp_version,omitempty"`
|
||||
SysName string `json:"sys_name"`
|
||||
SysObjectID string `json:"sys_object_id"`
|
||||
SysDescr string `json:"sys_descr"`
|
||||
PollState string `json:"poll_state"`
|
||||
Warnings []string `json:"warnings"`
|
||||
LastSeen *time.Time `json:"last_seen,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
RetiredAt *time.Time `json:"retired_at,omitempty"`
|
||||
Interfaces []InterfaceView `json:"interfaces"`
|
||||
}
|
||||
|
||||
type deviceOut struct{ Body DeviceView }
|
||||
|
||||
func (s *server) registerDevices(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "add-device", Method: "POST", Path: "/api/v1/devices", Tags: []string{"devices"},
|
||||
Summary: "Add a device",
|
||||
Description: "Creates the credential, device and addresses, runs seed discovery synchronously (30 s limit) and starts polling. A device that does not answer SNMP is rejected with 502 and nothing is stored.",
|
||||
DefaultStatus: 201,
|
||||
Errors: []int{409, 422, 502},
|
||||
}, s.addDevice)
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "get-device", Method: "GET", Path: "/api/v1/devices/{slug}", Tags: []string{"devices"},
|
||||
Summary: "Get a device and its interfaces",
|
||||
Errors: []int{404},
|
||||
}, s.getDevice)
|
||||
}
|
||||
|
||||
func (s *server) addDevice(ctx context.Context, in *struct{ Body AddDeviceBody }) (*deviceOut, error) {
|
||||
b := in.Body
|
||||
kind := b.Kind
|
||||
if kind == "" {
|
||||
kind = store.KindNetwork
|
||||
}
|
||||
var addrs []netip.Addr
|
||||
for _, a := range b.Addresses {
|
||||
ip, err := netip.ParseAddr(a)
|
||||
if err != nil {
|
||||
return nil, huma.Error422UnprocessableEntity(fmt.Sprintf("address %q is not an IP address", a))
|
||||
}
|
||||
addrs = append(addrs, ip.WithZone("").Unmap())
|
||||
}
|
||||
if len(addrs) == 0 {
|
||||
return nil, huma.Error422UnprocessableEntity("at least one address is required")
|
||||
}
|
||||
if kind == store.KindNetwork && b.SNMP == nil {
|
||||
return nil, huma.Error422UnprocessableEntity("snmp credentials are required for kind network")
|
||||
}
|
||||
var (
|
||||
cred *store.SNMPCredential
|
||||
res *discovery.Result
|
||||
)
|
||||
if b.SNMP != nil {
|
||||
var err error
|
||||
if cred, err = s.credential(b.SNMP); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pref := addrs[0]
|
||||
for _, a := range addrs {
|
||||
if a.Is6() {
|
||||
pref = a
|
||||
break
|
||||
}
|
||||
}
|
||||
if res, err = s.collect(ctx, pref, cred); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var slug string
|
||||
err := s.Store.WithTx(ctx, func(tx store.Store) error {
|
||||
d := &store.Device{Name: b.Name, Kind: kind, Site: b.Site, Role: b.Role, Enabled: true, Profile: "generic"}
|
||||
if cred != nil {
|
||||
if err := tx.CreateCredential(ctx, cred); err != nil {
|
||||
return err
|
||||
}
|
||||
d.SNMPCredentialID = &cred.ID
|
||||
}
|
||||
if err := tx.CreateDevice(ctx, d); err != nil {
|
||||
return err
|
||||
}
|
||||
strs := make([]string, len(addrs))
|
||||
for i, a := range addrs {
|
||||
strs[i] = a.String()
|
||||
}
|
||||
if err := tx.SetAddresses(ctx, d.ID, strs); err != nil {
|
||||
return err
|
||||
}
|
||||
if res != nil {
|
||||
if err := discovery.Apply(ctx, tx, d, res, s.Clock); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
slug = d.Slug
|
||||
return tx.AppendAudit(ctx, "anonymous", "device.add", d.Slug, nil,
|
||||
map[string]any{"name": d.Name, "kind": d.Kind, "addresses": strs})
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
s.Poller.Refresh()
|
||||
return s.view(ctx, slug)
|
||||
}
|
||||
|
||||
// credential validates the request's SNMP block and returns a fresh credential.
|
||||
func (s *server) credential(b *SNMPBody) (*store.SNMPCredential, error) {
|
||||
c := &store.SNMPCredential{
|
||||
Name: "cred-" + ulid.New(s.Clock.Now()), Version: b.Version, Community: b.Community, V3User: b.V3User,
|
||||
V3AuthProto: b.V3AuthProto, V3AuthPass: b.V3AuthPass, V3PrivProto: b.V3PrivProto, V3PrivPass: b.V3PrivPass,
|
||||
V3Context: b.V3Context,
|
||||
}
|
||||
switch b.Version {
|
||||
case "v1", "v2c":
|
||||
if b.Community == "" {
|
||||
return nil, huma.Error422UnprocessableEntity("snmp.community is required for " + b.Version)
|
||||
}
|
||||
case "v3":
|
||||
if b.V3User == "" {
|
||||
return nil, huma.Error422UnprocessableEntity("snmp.v3_user is required for v3")
|
||||
}
|
||||
if _, err := snmp.ParseAuthProto(b.V3AuthProto); err != nil {
|
||||
return nil, huma.Error422UnprocessableEntity(err.Error())
|
||||
}
|
||||
if _, err := snmp.ParsePrivProto(b.V3PrivProto); err != nil {
|
||||
return nil, huma.Error422UnprocessableEntity(err.Error())
|
||||
}
|
||||
default:
|
||||
return nil, huma.Error422UnprocessableEntity("snmp.version must be v1, v2c or v3")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// collect runs the seed walk against addr before anything is stored.
|
||||
func (s *server) collect(ctx context.Context, addr netip.Addr, cred *store.SNMPCredential) (*discovery.Result, error) {
|
||||
dctx, cancel := context.WithTimeout(ctx, s.DiscoveryTimeout)
|
||||
defer cancel()
|
||||
sess, err := snmp.Dial(dctx, discovery.Target(addr, s.SNMPPort, cred, s.Clock), "discovery")
|
||||
if err != nil {
|
||||
return nil, huma.Error422UnprocessableEntity("snmp: " + err.Error())
|
||||
}
|
||||
defer sess.Close()
|
||||
res, err := discovery.Collect(dctx, sess, s.Clock)
|
||||
if err != nil {
|
||||
return nil, huma.Error502BadGateway("discovery of " + addr.String() + " failed: " + err.Error())
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (s *server) getDevice(ctx context.Context, in *struct {
|
||||
Slug string `path:"slug" doc:"Device slug."`
|
||||
}) (*deviceOut, error) {
|
||||
return s.view(ctx, in.Slug)
|
||||
}
|
||||
|
||||
func (s *server) view(ctx context.Context, slug string) (*deviceOut, error) {
|
||||
d, err := s.Store.GetDeviceBySlug(ctx, slug)
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
if d.PurgedAt != nil {
|
||||
return nil, huma.Error404NotFound("not found")
|
||||
}
|
||||
v := DeviceView{
|
||||
Slug: d.Slug, Name: d.Name, Kind: d.Kind, Site: d.Site, Role: d.Role, Profile: d.Profile, Enabled: d.Enabled,
|
||||
MgmtIP: d.MgmtIP, SysName: d.SysName, SysObjectID: d.SysObjectID, SysDescr: d.SysDescr,
|
||||
PollState: d.PollState, Warnings: nonNil(d.Warnings), LastSeen: utcp(d.LastSeen), CreatedAt: d.CreatedAt.UTC(),
|
||||
RetiredAt: utcp(d.RetiredAt), Addresses: []string{}, Interfaces: []InterfaceView{},
|
||||
}
|
||||
addrs, err := s.Store.ListAddresses(ctx, d.ID)
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
for _, a := range addrs {
|
||||
v.Addresses = append(v.Addresses, a.Addr)
|
||||
}
|
||||
if d.SNMPCredentialID != nil {
|
||||
c, err := s.Store.GetCredential(ctx, *d.SNMPCredentialID)
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
v.SNMPVersion = c.Version // the secrets stay behind
|
||||
}
|
||||
ifs, err := s.Store.ListInterfaces(ctx, d.ID, false)
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
for _, i := range ifs {
|
||||
v.Interfaces = append(v.Interfaces, InterfaceView{
|
||||
ID: i.ID, IfIndex: i.IfIndex, IfName: i.IfName, IfDescr: i.IfDescr, IfAlias: i.IfAlias,
|
||||
IfPhysAddr: i.IfPhysAddr, IfSpeed: i.IfSpeed, IfHighSpeed: i.IfHighSpeed, IfType: i.IfType,
|
||||
AdminStatus: i.AdminStatus, OperStatus: i.OperStatus, HasHCCounters: i.HasHCCounters, Excluded: i.Excluded,
|
||||
FirstSeen: i.FirstSeen.UTC(), LastDiscovered: i.LastDiscovered.UTC(),
|
||||
})
|
||||
}
|
||||
return &deviceOut{Body: v}, nil
|
||||
}
|
||||
|
||||
func nonNil(s []string) []string {
|
||||
if s == nil {
|
||||
return []string{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func utcp(t *time.Time) *time.Time {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
u := t.UTC()
|
||||
return &u
|
||||
}
|
||||
29
internal/api/errors_test.go
Normal file
29
internal/api/errors_test.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
func TestStoreErrorMapping(t *testing.T) {
|
||||
s := &server{}
|
||||
s.Logger = nil
|
||||
for _, tc := range []struct {
|
||||
err error
|
||||
status int
|
||||
detail string
|
||||
}{
|
||||
{fmt.Errorf("%w: slug taken", store.ErrConflict), 409, "slug taken"},
|
||||
{fmt.Errorf("%w: bad kind", store.ErrInvalid), 422, "bad kind"},
|
||||
{fmt.Errorf("wrapped: %w", store.ErrNotFound), 404, "not found"},
|
||||
} {
|
||||
se, ok := s.storeErr(tc.err).(huma.StatusError)
|
||||
if !ok || se.GetStatus() != tc.status || se.Error() != tc.detail {
|
||||
t.Errorf("%v: got %v, want %d %q", tc.err, se, tc.status, tc.detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
169
internal/api/query.go
Normal file
169
internal/api/query.go
Normal file
@@ -0,0 +1,169 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/vmsup"
|
||||
)
|
||||
|
||||
var jsonObject = map[string]*huma.Response{
|
||||
"200": {Description: "The store's Prometheus-compatible JSON response, unchanged.", Content: map[string]*huma.MediaType{
|
||||
"application/json": {Schema: &huma.Schema{Type: "object"}},
|
||||
}},
|
||||
}
|
||||
|
||||
func (s *server) registerQuery(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "query", Method: "GET", Path: "/api/v1/query", Tags: []string{"query"},
|
||||
Summary: "Instant MetricsQL query against the hot store",
|
||||
Description: "Read-only proxy with a 30 s timeout. A result of more than 100,000 series is rejected with 422.",
|
||||
Responses: jsonObject, Errors: []int{422, 503, 504},
|
||||
}, s.query)
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "query-range", Method: "GET", Path: "/api/v1/query_range", Tags: []string{"query"},
|
||||
Summary: "Range MetricsQL query against the hot store",
|
||||
Description: "Same limits as /api/v1/query.",
|
||||
Responses: jsonObject, Errors: []int{422, 503, 504},
|
||||
}, s.queryRange)
|
||||
}
|
||||
|
||||
func raw(body []byte) *huma.StreamResponse {
|
||||
return &huma.StreamResponse{Body: func(ctx huma.Context) {
|
||||
ctx.SetHeader("Content-Type", "application/json")
|
||||
ctx.SetStatus(http.StatusOK)
|
||||
_, _ = ctx.BodyWriter().Write(body)
|
||||
}}
|
||||
}
|
||||
|
||||
func (s *server) query(ctx context.Context, in *struct {
|
||||
Q string `query:"q" required:"true" minLength:"1" doc:"MetricsQL expression." example:"ifHCInOctets"`
|
||||
Time string `query:"time" doc:"Evaluation time: RFC 3339 or Unix seconds. Default now."`
|
||||
}) (*huma.StreamResponse, error) {
|
||||
var ts time.Time
|
||||
if in.Time != "" {
|
||||
var err error
|
||||
if ts, err = parseTime(in.Time); err != nil {
|
||||
return nil, huma.Error422UnprocessableEntity("time: " + err.Error())
|
||||
}
|
||||
}
|
||||
qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
|
||||
defer cancel()
|
||||
body, err := s.Querier.Query(qctx, "hot", in.Q, ts)
|
||||
return s.finish(qctx, body, err)
|
||||
}
|
||||
|
||||
func (s *server) queryRange(ctx context.Context, in *struct {
|
||||
Q string `query:"q" required:"true" minLength:"1" doc:"MetricsQL expression."`
|
||||
From string `query:"from" doc:"Start: RFC 3339 or Unix seconds. Default one hour before to."`
|
||||
To string `query:"to" doc:"End: RFC 3339 or Unix seconds. Default now."`
|
||||
Step string `query:"step" required:"true" doc:"Resolution step: a duration such as 30s or a number of seconds."`
|
||||
}) (*huma.StreamResponse, error) {
|
||||
to := s.Clock.Now()
|
||||
if in.To != "" {
|
||||
var err error
|
||||
if to, err = parseTime(in.To); err != nil {
|
||||
return nil, huma.Error422UnprocessableEntity("to: " + err.Error())
|
||||
}
|
||||
}
|
||||
from := to.Add(-time.Hour)
|
||||
if in.From != "" {
|
||||
var err error
|
||||
if from, err = parseTime(in.From); err != nil {
|
||||
return nil, huma.Error422UnprocessableEntity("from: " + err.Error())
|
||||
}
|
||||
}
|
||||
step, err := parseStep(in.Step)
|
||||
if err != nil {
|
||||
return nil, huma.Error422UnprocessableEntity("step: " + err.Error())
|
||||
}
|
||||
if !from.Before(to) {
|
||||
return nil, huma.Error422UnprocessableEntity("from must be before to")
|
||||
}
|
||||
qctx, cancel := context.WithTimeout(ctx, QueryTimeout)
|
||||
defer cancel()
|
||||
body, err := s.Querier.QueryRange(qctx, "hot", in.Q, from, to, step)
|
||||
return s.finish(qctx, body, err)
|
||||
}
|
||||
|
||||
// finish applies the series cap and maps store errors to problems.
|
||||
func (s *server) finish(ctx context.Context, body []byte, err error) (*huma.StreamResponse, error) {
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.Is(err, vmsup.ErrStoreDown):
|
||||
return nil, huma.Error503ServiceUnavailable("the metric store is not available")
|
||||
case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded):
|
||||
return nil, huma.Error504GatewayTimeout("query exceeded " + QueryTimeout.String())
|
||||
case strings.Contains(err.Error(), "400 Bad Request") || strings.Contains(err.Error(), "422 Unprocessable"):
|
||||
return nil, huma.Error422UnprocessableEntity(err.Error()) // the store rejected the query
|
||||
default:
|
||||
s.Logger.Warn("query failed", "err", err)
|
||||
return nil, huma.Error502BadGateway("metric store error")
|
||||
}
|
||||
if n := countSeries(body); n > MaxSeries {
|
||||
return nil, huma.Error422UnprocessableEntity("query returned " + itoa(n) + " series; the limit is " + itoa(MaxSeries) + ", narrow the selector")
|
||||
}
|
||||
return raw(body), nil
|
||||
}
|
||||
|
||||
// countSeries returns len(data.result) of a Prometheus-style response.
|
||||
func countSeries(body []byte) int {
|
||||
var r struct {
|
||||
Data struct {
|
||||
Result []json.RawMessage `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if json.NewDecoder(bytes.NewReader(body)).Decode(&r) != nil {
|
||||
return 0
|
||||
}
|
||||
return len(r.Data.Result)
|
||||
}
|
||||
|
||||
func parseTime(s string) (time.Time, error) {
|
||||
if t, err := time.Parse(time.RFC3339, s); err == nil {
|
||||
return t.UTC(), nil
|
||||
}
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, errors.New("want RFC 3339 or Unix seconds")
|
||||
}
|
||||
return time.UnixMilli(int64(f * 1000)).UTC(), nil
|
||||
}
|
||||
|
||||
func parseStep(s string) (time.Duration, error) {
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil {
|
||||
f, ferr := strconv.ParseFloat(s, 64)
|
||||
if ferr != nil {
|
||||
return 0, errors.New("want a duration such as 30s or a number of seconds")
|
||||
}
|
||||
d = time.Duration(f * float64(time.Second))
|
||||
}
|
||||
if d <= 0 {
|
||||
return 0, errors.New("must be positive")
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (s *server) registerMetrics(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "metrics", Method: "GET", Path: "/metrics", Tags: []string{"internals"},
|
||||
Summary: "nmon's own metrics in Prometheus text format",
|
||||
Responses: map[string]*huma.Response{"200": {Description: "Prometheus text exposition.", Content: map[string]*huma.MediaType{
|
||||
"text/plain": {Schema: &huma.Schema{Type: "string"}},
|
||||
}}},
|
||||
}, func(context.Context, *struct{}) (*huma.StreamResponse, error) {
|
||||
return &huma.StreamResponse{Body: func(ctx huma.Context) {
|
||||
ctx.SetHeader("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
_ = s.Registry.WritePrometheus(ctx.BodyWriter())
|
||||
}}, nil
|
||||
})
|
||||
}
|
||||
263
internal/app/app.go
Normal file
263
internal/app/app.go
Normal file
@@ -0,0 +1,263 @@
|
||||
// Package app wires nmon serve: config, secret key, inventory, metric stores,
|
||||
// write queues, poller and HTTP API, and shuts them down in order.
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
"github.com/v10networks/nmon/internal/poller"
|
||||
"github.com/v10networks/nmon/internal/secrets"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
"github.com/v10networks/nmon/internal/vmsup"
|
||||
)
|
||||
|
||||
// Defaults for flags left empty.
|
||||
const (
|
||||
DefaultListen = "[::]:8443"
|
||||
DefaultSecretKeyFile = "/etc/nmon/secret.key"
|
||||
)
|
||||
|
||||
const (
|
||||
startTimeout = 60 * time.Second
|
||||
shutdownTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// App is a running nmon server.
|
||||
type App struct {
|
||||
log *slog.Logger
|
||||
st store.Store
|
||||
sup *vmsup.Supervisor
|
||||
hot *vmsup.Queue
|
||||
long *vmsup.Queue
|
||||
poller *poller.Poller
|
||||
srv *http.Server
|
||||
ln net.Listener
|
||||
|
||||
bg context.CancelFunc // poller and queue goroutines
|
||||
pollDone chan struct{}
|
||||
queuesWG chan struct{}
|
||||
serveErr chan error
|
||||
}
|
||||
|
||||
// Run starts nmon and blocks until ctx is cancelled or the HTTP server fails,
|
||||
// then shuts everything down gracefully.
|
||||
func Run(ctx context.Context, f config.Flags) error {
|
||||
a, err := Start(ctx, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var runErr error
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case runErr = <-a.serveErr:
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
defer cancel()
|
||||
return errors.Join(runErr, a.Stop(sctx))
|
||||
}
|
||||
|
||||
// Start brings every component up and returns once the API is listening. ctx
|
||||
// bounds startup only; call Stop to shut down.
|
||||
func Start(ctx context.Context, f config.Flags) (_ *App, err error) {
|
||||
log := slog.Default()
|
||||
if !f.InsecureHTTP {
|
||||
return nil, errors.New("TLS and authentication arrive in M6; until then pass --insecure-http to run without them (development only)")
|
||||
}
|
||||
if f.DataDir == "" {
|
||||
return nil, errors.New("--data-dir is required")
|
||||
}
|
||||
if f.Listen == "" {
|
||||
f.Listen = DefaultListen
|
||||
}
|
||||
if f.SecretKeyFile == "" {
|
||||
f.SecretKeyFile = DefaultSecretKeyFile
|
||||
}
|
||||
if f.MaxInflight <= 0 {
|
||||
f.MaxInflight = poller.DefaultMaxInflight
|
||||
}
|
||||
dataDir, err := filepath.Abs(f.DataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.DataDir = dataDir
|
||||
keyFile, err := filepath.Abs(f.SecretKeyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if inside(dataDir, keyFile) {
|
||||
return nil, fmt.Errorf("--secret-key-file %s is inside --data-dir %s; a backup of the data dir must not contain the key", keyFile, dataDir)
|
||||
}
|
||||
if err := os.MkdirAll(dataDir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg, err := config.Load(dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
driver := f.DBDriver
|
||||
if driver == "" {
|
||||
driver = store.DriverSQLite
|
||||
}
|
||||
dsn := f.DBDSN
|
||||
if dsn == "" {
|
||||
if driver != store.DriverSQLite {
|
||||
return nil, fmt.Errorf("--db-dsn is required for --db-driver %s", driver)
|
||||
}
|
||||
dsn = config.InventoryDB(dataDir)
|
||||
}
|
||||
if err := ensureKey(log, keyFile, driver == store.DriverSQLite && fileExists(config.InventoryDB(dataDir))); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kr, err := secrets.LoadKeyring(keyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("secret key: %w", err)
|
||||
}
|
||||
|
||||
a := &App{log: log, serveErr: make(chan error, 1), pollDone: make(chan struct{}), queuesWG: make(chan struct{})}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
defer cancel()
|
||||
_ = a.Stop(sctx)
|
||||
}
|
||||
}()
|
||||
|
||||
if a.st, err = store.Open(ctx, driver, dsn, kr, clock.Real()); err != nil {
|
||||
return nil, fmt.Errorf("open inventory: %w", err)
|
||||
}
|
||||
if err = a.st.Migrate(ctx); err != nil {
|
||||
return nil, fmt.Errorf("migrate inventory: %w", err)
|
||||
}
|
||||
|
||||
a.sup, err = vmsup.New(vmsup.Options{
|
||||
DataDir: dataDir, HotURL: f.VMHotURL, LongURL: f.VMLongURL,
|
||||
HotRetention: cfg.Retention.Hot.D(), LongRetention: cfg.Retention.Long.D(), Logger: log,
|
||||
OnEvent: func(e vmsup.Event) {
|
||||
log.Warn("metric store event", "type", e.Type, "store", e.Store, "detail", e.Detail, "restarts", e.Restarts)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(ctx, startTimeout)
|
||||
defer cancel()
|
||||
if err = a.sup.Start(sctx); err != nil {
|
||||
a.sup = nil // Start leaves nothing running when it fails
|
||||
return nil, fmt.Errorf("start metric stores: %w", err)
|
||||
}
|
||||
log.Info("metric stores up", "hot", a.sup.HotURL(), "long", a.sup.LongURL())
|
||||
|
||||
bg, cancelBG := context.WithCancel(context.Background())
|
||||
a.bg = cancelBG
|
||||
a.hot = vmsup.NewQueue(vmsup.StoreHot, a.sup, vmsup.QueueOptions{})
|
||||
a.long = vmsup.NewQueue(vmsup.StoreLong, a.sup, vmsup.QueueOptions{})
|
||||
qdone := make(chan struct{}, 2)
|
||||
for _, q := range []*vmsup.Queue{a.hot, a.long} {
|
||||
go func() { q.Run(bg); qdone <- struct{}{} }()
|
||||
}
|
||||
go func() { <-qdone; <-qdone; close(a.queuesWG) }()
|
||||
|
||||
a.poller = poller.New(poller.Options{
|
||||
Store: a.st, Writer: a.hot, Interval: f.PollInterval, MaxInflight: f.MaxInflight, SNMPPort: f.SNMPPort, Logger: log,
|
||||
})
|
||||
go func() { _ = a.poller.Run(bg); close(a.pollDone) }()
|
||||
|
||||
handler := api.New(api.Deps{
|
||||
Store: a.st, Querier: vmsup.NewQuerier(a.sup), Poller: a.poller, SNMPPort: f.SNMPPort, Logger: log,
|
||||
})
|
||||
if a.ln, err = net.Listen("tcp", f.Listen); err != nil {
|
||||
return nil, fmt.Errorf("listen: %w", err)
|
||||
}
|
||||
a.srv = &http.Server{Handler: handler, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
|
||||
go func() {
|
||||
if err := a.srv.Serve(a.ln); !errors.Is(err, http.ErrServerClosed) {
|
||||
a.serveErr <- err
|
||||
}
|
||||
}()
|
||||
log.Warn("serving plain HTTP without authentication (--insecure-http); do not expose this port", "addr", a.ln.Addr().String())
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Addr is the address the API listens on.
|
||||
func (a *App) Addr() string { return a.ln.Addr().String() }
|
||||
|
||||
// HotURL is the base URL of the hot metric store.
|
||||
func (a *App) HotURL() string { return a.sup.HotURL() }
|
||||
|
||||
// Stop shuts down: HTTP first, then the poller, then the metric stores (which
|
||||
// flushes the write queues), then the inventory. It is safe after a failed Start.
|
||||
func (a *App) Stop(ctx context.Context) error {
|
||||
var errs []error
|
||||
if a.srv != nil {
|
||||
errs = append(errs, a.srv.Shutdown(ctx))
|
||||
}
|
||||
if a.bg != nil {
|
||||
a.bg()
|
||||
<-a.pollDone
|
||||
<-a.queuesWG
|
||||
}
|
||||
if a.sup != nil {
|
||||
errs = append(errs, a.sup.Stop(ctx)) // flushes both queues
|
||||
}
|
||||
if a.st != nil {
|
||||
errs = append(errs, a.st.Close())
|
||||
}
|
||||
a.log.Info("stopped")
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// ensureKey creates the key file on a first run. It refuses when an inventory
|
||||
// already exists, since a new key could not decrypt its credentials.
|
||||
func ensureKey(log *slog.Logger, path string, inventoryExists bool) error {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return nil
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if inventoryExists {
|
||||
return fmt.Errorf("secret key file %s is missing but the inventory exists; restore the key file (a new key cannot decrypt the stored SNMP credentials)", path)
|
||||
}
|
||||
if err := secrets.GenerateKeyFile(path); err != nil {
|
||||
return fmt.Errorf("generate secret key: %w", err)
|
||||
}
|
||||
log.Error("GENERATED A NEW SECRET KEY: back it up now; without it the SNMP credentials in the inventory cannot be recovered", "path", path)
|
||||
return nil
|
||||
}
|
||||
|
||||
// inside reports whether path is dir or below it.
|
||||
func inside(dir, path string) bool {
|
||||
rel, err := filepath.Rel(resolve(dir), resolve(path))
|
||||
return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
// resolve follows symlinks in the deepest existing parent of p.
|
||||
func resolve(p string) string {
|
||||
rest := ""
|
||||
for q := p; ; q = filepath.Dir(q) {
|
||||
if r, err := filepath.EvalSymlinks(q); err == nil {
|
||||
return filepath.Join(r, rest)
|
||||
}
|
||||
if filepath.Dir(q) == q {
|
||||
return p
|
||||
}
|
||||
rest = filepath.Join(filepath.Base(q), rest)
|
||||
}
|
||||
}
|
||||
|
||||
func fileExists(p string) bool {
|
||||
_, err := os.Stat(p)
|
||||
return err == nil
|
||||
}
|
||||
94
internal/app/app_test.go
Normal file
94
internal/app/app_test.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
"github.com/v10networks/nmon/internal/secrets"
|
||||
)
|
||||
|
||||
func TestStartRefusals(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
data := filepath.Join(t.TempDir(), "data")
|
||||
outside := filepath.Join(t.TempDir(), "secret.key")
|
||||
for name, tc := range map[string]struct {
|
||||
f config.Flags
|
||||
want string
|
||||
}{
|
||||
"no insecure flag": {config.Flags{DataDir: data, SecretKeyFile: outside}, "--insecure-http"},
|
||||
"no data dir": {config.Flags{InsecureHTTP: true, SecretKeyFile: outside}, "--data-dir"},
|
||||
"key inside data": {config.Flags{DataDir: data, SecretKeyFile: filepath.Join(data, "keys", "secret.key"), InsecureHTTP: true}, "inside --data-dir"},
|
||||
"key is data dir": {config.Flags{DataDir: data, SecretKeyFile: data, InsecureHTTP: true}, "inside --data-dir"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
a, err := Start(ctx, tc.f)
|
||||
if err == nil {
|
||||
a.Stop(ctx)
|
||||
t.Fatal("started")
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("error %q, want it to mention %q", err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := os.Stat(outside); err == nil {
|
||||
t.Error("a refused start created the key file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartRefusesMissingKeyWithExistingInventory(t *testing.T) {
|
||||
data := t.TempDir()
|
||||
if err := os.WriteFile(config.InventoryDB(data), nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := filepath.Join(t.TempDir(), "secret.key")
|
||||
_, err := Start(context.Background(), config.Flags{DataDir: data, SecretKeyFile: key, InsecureHTTP: true})
|
||||
if err == nil || !strings.Contains(err.Error(), "inventory exists") {
|
||||
t.Fatalf("error %v", err)
|
||||
}
|
||||
if _, err := os.Stat(key); err == nil {
|
||||
t.Error("a new key was generated next to an existing inventory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureKeyGeneratesOnce(t *testing.T) {
|
||||
key := filepath.Join(t.TempDir(), "sub", "secret.key")
|
||||
if err := ensureKey(testLogger(), key, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := os.Stat(key)
|
||||
if err != nil || st.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("key file: %v %v", st, err)
|
||||
}
|
||||
if _, err := secrets.LoadKeyring(key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := os.ReadFile(key)
|
||||
if err := ensureKey(testLogger(), key, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after, _ := os.ReadFile(key); string(after) != string(before) {
|
||||
t.Error("existing key was replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInside(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
dir, path string
|
||||
want bool
|
||||
}{
|
||||
{"/var/lib/nmon", "/var/lib/nmon/secret.key", true},
|
||||
{"/var/lib/nmon", "/var/lib/nmon", true},
|
||||
{"/var/lib/nmon", "/var/lib/nmon2/secret.key", false},
|
||||
{"/var/lib/nmon", "/etc/nmon/secret.key", false},
|
||||
{"/var/lib/nmon", "/var/lib/secret.key", false},
|
||||
} {
|
||||
if got := inside(tc.dir, tc.path); got != tc.want {
|
||||
t.Errorf("inside(%s, %s) = %v", tc.dir, tc.path, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
8
internal/app/helpers_test.go
Normal file
8
internal/app/helpers_test.go
Normal file
@@ -0,0 +1,8 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
func testLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }
|
||||
175
internal/app/integration_test.go
Normal file
175
internal/app/integration_test.go
Normal file
@@ -0,0 +1,175 @@
|
||||
//go:build integration
|
||||
|
||||
package app_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/app"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
"github.com/v10networks/nmon/internal/simagent"
|
||||
"github.com/v10networks/nmon/internal/testutil"
|
||||
"github.com/v10networks/nmon/internal/vmsup"
|
||||
)
|
||||
|
||||
type series struct {
|
||||
Metric map[string]string `json:"metric"`
|
||||
Value [2]any `json:"value"`
|
||||
}
|
||||
|
||||
func query(t *testing.T, base, q string) []series {
|
||||
t.Helper()
|
||||
resp, err := http.Get(base + "/api/v1/query?q=" + q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("query %s: %d %s", q, resp.StatusCode, body)
|
||||
}
|
||||
var r struct {
|
||||
Data struct{ Result []series } `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r.Data.Result
|
||||
}
|
||||
|
||||
func TestServeCollectsCountersEndToEnd(t *testing.T) {
|
||||
rec, err := simagent.LoadRecording("../../testdata/snmprec/counters.snmprec")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
agent := testutil.StartAgent(t, simagent.Config{Recording: rec, Community: "public"})
|
||||
|
||||
root := t.TempDir()
|
||||
data := filepath.Join(root, "data")
|
||||
a, err := app.Start(context.Background(), config.Flags{
|
||||
DataDir: data, SecretKeyFile: filepath.Join(root, "keys", "secret.key"), Listen: "[::1]:0", InsecureHTTP: true,
|
||||
PollInterval: 5 * time.Second, SNMPPort: int(agent.Addr().Port()),
|
||||
})
|
||||
if errors.Is(err, vmsup.ErrNoBinary) {
|
||||
t.Skip("built without the vmembed tag")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stopped := false
|
||||
stop := func() {
|
||||
if stopped {
|
||||
return
|
||||
}
|
||||
stopped = true
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
|
||||
defer cancel()
|
||||
if err := a.Stop(ctx); err != nil {
|
||||
t.Errorf("stop: %v", err)
|
||||
}
|
||||
}
|
||||
t.Cleanup(stop)
|
||||
base := "http://" + a.Addr()
|
||||
|
||||
resp, err := http.Post(base+"/api/v1/devices", "application/json", strings.NewReader(
|
||||
`{"name":"counters","addresses":["::1"],"snmp":{"version":"v2c","community":"public"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 201 {
|
||||
t.Fatalf("add device: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
var dev struct {
|
||||
Slug string
|
||||
Interfaces []struct{ ID string }
|
||||
}
|
||||
if err := json.Unmarshal(body, &dev); err != nil || len(dev.Interfaces) != 8 {
|
||||
t.Fatalf("device: %s (%v)", body, err)
|
||||
}
|
||||
|
||||
// Instant queries return the newest point, so collect values of one counter
|
||||
// across polls until three distinct ones have been seen.
|
||||
var vals []float64
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) && len(distinct(vals)) < 3 {
|
||||
time.Sleep(2 * time.Second)
|
||||
if r, err := http.Post(a.HotURL()+"/internal/force_flush", "", nil); err == nil {
|
||||
r.Body.Close()
|
||||
}
|
||||
for _, s := range query(t, base, "ifHCInOctets") {
|
||||
if s.Metric["iface"] == dev.Interfaces[0].ID {
|
||||
v, _ := strconv.ParseFloat(s.Value[1].(string), 64)
|
||||
vals = append(vals, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
d := distinct(vals)
|
||||
if len(d) < 3 {
|
||||
t.Fatalf("saw only %v for ifHCInOctets after 60s", d)
|
||||
}
|
||||
for i := 1; i < len(d); i++ {
|
||||
if d[i] <= d[i-1] {
|
||||
t.Fatalf("not rising: %v", d)
|
||||
}
|
||||
}
|
||||
for _, s := range query(t, base, "ifHCInOctets") {
|
||||
keys := make([]string, 0, len(s.Metric))
|
||||
for k := range s.Metric {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
if fmt.Sprint(keys) != "[__name__ device iface]" || s.Metric["device"] != dev.Slug {
|
||||
t.Fatalf("labels %v", s.Metric)
|
||||
}
|
||||
}
|
||||
if up := query(t, base, "nmon_snmp_up"); len(up) != 1 || up[0].Value[1] != "1" {
|
||||
t.Errorf("nmon_snmp_up = %+v", up)
|
||||
}
|
||||
// the poller stores nmon_ internals every 15 s
|
||||
stored := false
|
||||
for end := time.Now().Add(30 * time.Second); time.Now().Before(end) && !stored; time.Sleep(2 * time.Second) {
|
||||
if r, err := http.Post(a.HotURL()+"/internal/force_flush", "", nil); err == nil {
|
||||
r.Body.Close()
|
||||
}
|
||||
stored = len(query(t, base, "nmon_pdus_total")) > 0
|
||||
}
|
||||
if !stored {
|
||||
t.Error("nmon_ internals are not stored")
|
||||
}
|
||||
if r, err := http.Get(base + "/metrics"); err != nil || r.StatusCode != 200 {
|
||||
t.Errorf("/metrics: %v %v", r, err)
|
||||
} else {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
r.Body.Close()
|
||||
if !bytes.Contains(b, []byte("nmon_pdus_total")) {
|
||||
t.Error("/metrics lacks nmon_pdus_total")
|
||||
}
|
||||
}
|
||||
stop()
|
||||
}
|
||||
|
||||
func distinct(v []float64) []float64 {
|
||||
s := append([]float64(nil), v...)
|
||||
sort.Float64s(s)
|
||||
out := s[:0]
|
||||
for i, x := range s {
|
||||
if i == 0 || x != s[i-1] {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -25,6 +25,7 @@ type Flags struct {
|
||||
VMLongURL string
|
||||
MaxInflight int
|
||||
PollInterval time.Duration // dev/test override of the 60 s interface interval; 0 = profile default
|
||||
SNMPPort int // dev/test override of UDP 161 for every device; 0 = 161
|
||||
}
|
||||
|
||||
// Config is the content of config.yaml.
|
||||
|
||||
255
internal/discovery/discovery.go
Normal file
255
internal/discovery/discovery.go
Normal file
@@ -0,0 +1,255 @@
|
||||
// Package discovery learns what a device has. In M1 it is the seed walk run
|
||||
// when a device is added: system group plus the IF-MIB tables, creating
|
||||
// interface rows with ULIDs.
|
||||
//
|
||||
// TODO(M2): identity matching beyond ifName and ifDescr+MAC (stack-member
|
||||
// swaps, RMA), identity_reviews for ambiguous matches, retiring interfaces that
|
||||
// disappeared, profiles, per-table limits and the `partial` flag.
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/snmp"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
"github.com/v10networks/nmon/internal/ulid"
|
||||
)
|
||||
|
||||
// Walk roots.
|
||||
const (
|
||||
oidSystem = "1.3.6.1.2.1.1"
|
||||
oidIfTable = "1.3.6.1.2.1.2.2.1"
|
||||
oidIfXTable = "1.3.6.1.2.1.31.1.1.1"
|
||||
)
|
||||
|
||||
// Target builds the SNMP target for a device's credential. port 0 means 161.
|
||||
func Target(addr netip.Addr, port int, c *store.SNMPCredential, clk clock.Clock) snmp.Target {
|
||||
t := snmp.Target{Addr: netip.AddrPortFrom(addr, uint16(port)), Clock: clk}
|
||||
t.Cred = snmp.Credential{
|
||||
Community: c.Community, V3User: c.V3User, V3AuthProto: c.V3AuthProto, V3AuthPass: c.V3AuthPass,
|
||||
V3PrivProto: c.V3PrivProto, V3PrivPass: c.V3PrivPass, V3Context: c.V3Context,
|
||||
}
|
||||
t.Cred.Version = strings.TrimPrefix(c.Version, "v")
|
||||
return t
|
||||
}
|
||||
|
||||
// Iface is one interface as the device reports it.
|
||||
type Iface struct {
|
||||
IfIndex int64
|
||||
Name, Descr string
|
||||
Alias string
|
||||
PhysAddr string // colon-separated lower-case hex
|
||||
Speed int64
|
||||
HighSpeed int64
|
||||
Type int64
|
||||
Admin, Oper int64
|
||||
HasHC bool
|
||||
}
|
||||
|
||||
// Result is what Collect learned.
|
||||
type Result struct {
|
||||
SysDescr, SysObjectID, SysName string
|
||||
SysUpTime time.Duration
|
||||
Ifaces []Iface
|
||||
StartedAt, FinishedAt time.Time
|
||||
}
|
||||
|
||||
// Collect walks the system group and the interface tables over sess, which
|
||||
// must be a "discovery" session.
|
||||
func Collect(ctx context.Context, sess *snmp.Session, clk clock.Clock) (*Result, error) {
|
||||
res := &Result{StartedAt: clk.Now().UTC()}
|
||||
err := sess.Walk(ctx, oidSystem, 0, func(vb snmp.Varbind) error {
|
||||
switch vb.OID {
|
||||
case oidSystem + ".1.0":
|
||||
res.SysDescr = text(vb)
|
||||
case oidSystem + ".2.0":
|
||||
res.SysObjectID, _ = vb.Value.(string)
|
||||
case oidSystem + ".3.0":
|
||||
if v, ok := vb.Uint64(); ok {
|
||||
res.SysUpTime = time.Duration(v) * 10 * time.Millisecond
|
||||
}
|
||||
case oidSystem + ".5.0":
|
||||
res.SysName = text(vb)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walk system group: %w", err)
|
||||
}
|
||||
|
||||
ifs := map[int64]*Iface{}
|
||||
get := func(idx int64) *Iface {
|
||||
i := ifs[idx]
|
||||
if i == nil {
|
||||
i = &Iface{IfIndex: idx}
|
||||
ifs[idx] = i
|
||||
}
|
||||
return i
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
root string
|
||||
set func(*Iface, snmp.Varbind)
|
||||
}{
|
||||
{"ifIndex", oidIfTable + ".1", func(*Iface, snmp.Varbind) {}},
|
||||
{"ifDescr", oidIfTable + ".2", func(i *Iface, vb snmp.Varbind) { i.Descr = text(vb) }},
|
||||
{"ifType", oidIfTable + ".3", func(i *Iface, vb snmp.Varbind) { i.Type = num(vb) }},
|
||||
{"ifSpeed", oidIfTable + ".5", func(i *Iface, vb snmp.Varbind) { i.Speed = num(vb) }},
|
||||
{"ifPhysAddress", oidIfTable + ".6", func(i *Iface, vb snmp.Varbind) { i.PhysAddr = mac(vb) }},
|
||||
{"ifAdminStatus", oidIfTable + ".7", func(i *Iface, vb snmp.Varbind) { i.Admin = num(vb) }},
|
||||
{"ifOperStatus", oidIfTable + ".8", func(i *Iface, vb snmp.Varbind) { i.Oper = num(vb) }},
|
||||
{"ifName", oidIfXTable + ".1", func(i *Iface, vb snmp.Varbind) { i.Name = text(vb) }},
|
||||
{"ifHighSpeed", oidIfXTable + ".15", func(i *Iface, vb snmp.Varbind) { i.HighSpeed = num(vb) }},
|
||||
{"ifAlias", oidIfXTable + ".18", func(i *Iface, vb snmp.Varbind) { i.Alias = text(vb) }},
|
||||
{"ifHCInOctets", oidIfXTable + ".6", func(i *Iface, vb snmp.Varbind) { i.HasHC = true }},
|
||||
} {
|
||||
prefix := c.root + "."
|
||||
err := sess.Walk(ctx, c.root, 0, func(vb snmp.Varbind) error {
|
||||
idx, err := strconv.ParseInt(strings.TrimPrefix(vb.OID, prefix), 10, 64)
|
||||
if err != nil {
|
||||
return nil // not a plain single-index row
|
||||
}
|
||||
c.set(get(idx), vb)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("walk %s: %w", c.name, err)
|
||||
}
|
||||
}
|
||||
res.Ifaces = make([]Iface, 0, len(ifs))
|
||||
for _, i := range ifs {
|
||||
res.Ifaces = append(res.Ifaces, *i)
|
||||
}
|
||||
sortIfaces(res.Ifaces)
|
||||
res.FinishedAt = clk.Now().UTC()
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Seed discovers dev over sess (a "discovery" session) and records the result:
|
||||
// sys_* fields, interface rows and a discoveries row.
|
||||
func Seed(ctx context.Context, st store.Store, sess *snmp.Session, dev *store.Device, clk clock.Clock) error {
|
||||
res, err := Collect(ctx, sess, clk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return Apply(ctx, st, dev, res, clk)
|
||||
}
|
||||
|
||||
// Apply writes a Collect result for dev. Existing interfaces are matched by
|
||||
// ifName, then by ifDescr plus MAC, and keep their IDs; the rest get new ULIDs.
|
||||
// Interfaces the device no longer reports are left alone (TODO(M2): retire or review).
|
||||
func Apply(ctx context.Context, st store.Store, dev *store.Device, res *Result, clk clock.Clock) error {
|
||||
return st.WithTx(ctx, func(tx store.Store) error {
|
||||
existing, err := tx.ListInterfaces(ctx, dev.ID, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows := match(existing, res.Ifaces, clk.Now())
|
||||
// UpdateDevice writes every mutable column, so start from the stored row
|
||||
// rather than a caller's copy that may predate SetAddresses.
|
||||
cur, err := tx.GetDeviceByID(ctx, dev.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cur.SysDescr, cur.SysObjectID, cur.SysName = res.SysDescr, res.SysObjectID, res.SysName
|
||||
if cur.Profile == "" {
|
||||
cur.Profile = "generic"
|
||||
}
|
||||
if err := tx.UpdateDevice(ctx, cur); err != nil {
|
||||
return err
|
||||
}
|
||||
*dev = *cur
|
||||
if err := tx.UpsertInterfaces(ctx, dev.ID, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
tables, _ := json.Marshal(map[string]int{"interfaces": len(rows)})
|
||||
return tx.RecordDiscovery(ctx, &store.Discovery{
|
||||
DeviceID: dev.ID, StartedAt: res.StartedAt, FinishedAt: res.FinishedAt, TablesJSON: string(tables),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// match pairs discovered interfaces with existing rows and returns the rows to upsert.
|
||||
func match(existing []store.Interface, found []Iface, now time.Time) []store.Interface {
|
||||
byName := map[string][]int{}
|
||||
byDescrMAC := map[string][]int{}
|
||||
for k, e := range existing {
|
||||
if e.IfName != "" {
|
||||
byName[e.IfName] = append(byName[e.IfName], k)
|
||||
}
|
||||
key := e.IfDescr + "\x00" + e.IfPhysAddr
|
||||
byDescrMAC[key] = append(byDescrMAC[key], k)
|
||||
}
|
||||
used := make([]bool, len(existing))
|
||||
take := func(c []int) int {
|
||||
for _, k := range c {
|
||||
if !used[k] {
|
||||
used[k] = true
|
||||
return k
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
out := make([]store.Interface, 0, len(found))
|
||||
for _, f := range found {
|
||||
k := -1
|
||||
if f.Name != "" {
|
||||
k = take(byName[f.Name])
|
||||
}
|
||||
if k < 0 {
|
||||
k = take(byDescrMAC[f.Descr+"\x00"+f.PhysAddr])
|
||||
}
|
||||
var row store.Interface
|
||||
if k >= 0 {
|
||||
row = existing[k]
|
||||
} else {
|
||||
row = store.Interface{ID: ulid.New(now)}
|
||||
}
|
||||
row.IfIndex, row.IfName, row.IfDescr, row.IfAlias = f.IfIndex, f.Name, f.Descr, f.Alias
|
||||
row.IfPhysAddr, row.IfSpeed, row.IfHighSpeed, row.IfType = f.PhysAddr, f.Speed, f.HighSpeed, f.Type
|
||||
row.AdminStatus, row.OperStatus, row.HasHCCounters = f.Admin, f.Oper, f.HasHC
|
||||
out = append(out, row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortIfaces(s []Iface) {
|
||||
slices.SortFunc(s, func(a, b Iface) int { return cmp.Compare(a.IfIndex, b.IfIndex) })
|
||||
}
|
||||
func text(vb snmp.Varbind) string {
|
||||
switch v := vb.Value.(type) {
|
||||
case []byte:
|
||||
return strings.ToValidUTF8(string(v), "?")
|
||||
case string:
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func num(vb snmp.Varbind) int64 {
|
||||
v, _ := vb.Int64()
|
||||
return v
|
||||
}
|
||||
|
||||
func mac(vb snmp.Varbind) string {
|
||||
b, _ := vb.Value.([]byte)
|
||||
if len(b) == 0 {
|
||||
return ""
|
||||
}
|
||||
h := hex.EncodeToString(b)
|
||||
parts := make([]string, 0, len(b))
|
||||
for i := 0; i < len(h); i += 2 {
|
||||
parts = append(parts, h[i:i+2])
|
||||
}
|
||||
return strings.Join(parts, ":")
|
||||
}
|
||||
115
internal/discovery/discovery_test.go
Normal file
115
internal/discovery/discovery_test.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package discovery_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/fixturegen"
|
||||
"github.com/v10networks/nmon/internal/simagent"
|
||||
"github.com/v10networks/nmon/internal/testutil"
|
||||
"github.com/v10networks/nmon/internal/ulid"
|
||||
)
|
||||
|
||||
// shuffled returns rec with every interface index i replaced by 100+n+1-i, in
|
||||
// both ifTable and ifXTable, so every port keeps its name but changes ifIndex.
|
||||
func shuffled(rec *simagent.Recording, n int) *simagent.Recording {
|
||||
const ifT, ifX = "1.3.6.1.2.1.2.2.1.", "1.3.6.1.2.1.31.1.1.1."
|
||||
remap := func(i int) int { return 100 + n + 1 - i }
|
||||
var out []simagent.Record
|
||||
for _, r := range rec.Records() {
|
||||
for _, p := range []string{ifT, ifX} {
|
||||
if !strings.HasPrefix(r.OID, p) {
|
||||
continue
|
||||
}
|
||||
cut := strings.LastIndexByte(r.OID, '.')
|
||||
var i int
|
||||
for _, c := range r.OID[cut+1:] {
|
||||
i = i*10 + int(c-'0')
|
||||
}
|
||||
r.OID = r.OID[:cut+1] + strconv.Itoa(remap(i))
|
||||
if strings.HasPrefix(r.OID, ifT+"1.") {
|
||||
r.Value = remap(i)
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return simagent.NewRecording(out)
|
||||
}
|
||||
|
||||
func TestSeedCreatesAndKeepsInterfaceIDs(t *testing.T) {
|
||||
clk := clock.NewFake(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC))
|
||||
st := testutil.OpenStore(t, clk)
|
||||
ctx := context.Background()
|
||||
rec := fixturegen.Generic(fixturegen.Options{Ports: 8, SysName: "sw1", HCCounters: true})
|
||||
a := testutil.StartAgent(t, simagent.Config{Recording: rec})
|
||||
dev := testutil.AddDevice(t, st, a, "sw1", "public", clk)
|
||||
|
||||
if dev.SysName != "sw1" || dev.SysObjectID != "1.3.6.1.4.1.8072.3.2.10" || dev.SysDescr != "nmon generic fixture" {
|
||||
t.Fatalf("sys fields: %+v", dev)
|
||||
}
|
||||
ifs, err := st.ListInterfaces(ctx, dev.ID, false)
|
||||
if err != nil || len(ifs) != 8 {
|
||||
t.Fatalf("interfaces: %d, %v", len(ifs), err)
|
||||
}
|
||||
ids := map[string]string{}
|
||||
for _, i := range ifs {
|
||||
if !ulid.Valid(i.ID) {
|
||||
t.Errorf("bad ULID %q", i.ID)
|
||||
}
|
||||
if !i.HasHCCounters || i.IfName == "" || i.IfDescr == "" || i.IfSpeed == 0 && i.IfType == 6 {
|
||||
t.Errorf("incomplete interface %+v", i)
|
||||
}
|
||||
ids[i.IfName] = i.ID
|
||||
}
|
||||
if ifs[0].IfPhysAddr == "" || strings.Count(ifs[0].IfPhysAddr, ":") != 5 {
|
||||
t.Errorf("mac %q", ifs[0].IfPhysAddr)
|
||||
}
|
||||
d, err := st.LastDiscovery(ctx, dev.ID)
|
||||
if err != nil || d.Partial || d.TablesJSON == "{}" {
|
||||
t.Fatalf("discovery row: %+v %v", d, err)
|
||||
}
|
||||
|
||||
// Same device again with every ifIndex shuffled: IDs must follow the ifName.
|
||||
a2 := testutil.StartAgent(t, simagent.Config{Recording: shuffled(rec, 8)})
|
||||
cred, err := st.GetCredential(ctx, *dev.SNMPCredentialID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
testutil.Seed(t, st, a2, dev, cred, clk)
|
||||
after, err := st.ListInterfaces(ctx, dev.ID, false)
|
||||
if err != nil || len(after) != 8 {
|
||||
t.Fatalf("after reshuffle: %d, %v", len(after), err)
|
||||
}
|
||||
moved := 0
|
||||
for _, i := range after {
|
||||
if ids[i.IfName] != i.ID {
|
||||
t.Errorf("%s: id %s, was %s", i.IfName, i.ID, ids[i.IfName])
|
||||
}
|
||||
if i.IfIndex > 100 {
|
||||
moved++
|
||||
}
|
||||
}
|
||||
if moved != 8 {
|
||||
t.Errorf("%d interfaces carry the new ifIndex, want 8", moved)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedWithoutHCCounters(t *testing.T) {
|
||||
clk := clock.NewFake(time.Now())
|
||||
st := testutil.OpenStore(t, clk)
|
||||
a := testutil.StartAgent(t, simagent.Config{Recording: fixturegen.Generic(fixturegen.Options{Ports: 4})})
|
||||
dev := testutil.AddDevice(t, st, a, "old", "public", clk)
|
||||
ifs, _ := st.ListInterfaces(context.Background(), dev.ID, false)
|
||||
if len(ifs) != 4 {
|
||||
t.Fatalf("%d interfaces", len(ifs))
|
||||
}
|
||||
for _, i := range ifs {
|
||||
if i.HasHCCounters {
|
||||
t.Errorf("%s: has_hc_counters set", i.IfName)
|
||||
}
|
||||
}
|
||||
}
|
||||
299
internal/poller/cycle.go
Normal file
299
internal/poller/cycle.go
Normal file
@@ -0,0 +1,299 @@
|
||||
package poller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
|
||||
"github.com/v10networks/nmon/internal/discovery"
|
||||
"github.com/v10networks/nmon/internal/icmp"
|
||||
"github.com/v10networks/nmon/internal/metrics"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// kind says how a varbind becomes a sample value.
|
||||
type kind int
|
||||
|
||||
const (
|
||||
kindUptime kind = iota // TimeTicks to seconds
|
||||
kindCounter // unsigned
|
||||
kindStatus // integer enum
|
||||
)
|
||||
|
||||
// item is what one requested OID means.
|
||||
type item struct {
|
||||
name string // metric name
|
||||
iface string // interface ULID; empty for device-level series
|
||||
kind kind
|
||||
}
|
||||
|
||||
// target is a device as the loop polls it; it is replaced wholesale on reload.
|
||||
type target struct {
|
||||
dev store.Device
|
||||
addr netip.Addr
|
||||
cred *store.SNMPCredential // nil: the device has no SNMP credential
|
||||
oids []string
|
||||
items []item
|
||||
ifaces int
|
||||
}
|
||||
|
||||
const oidSysUpTime = "1.3.6.1.2.1.1.3.0"
|
||||
|
||||
// IF-MIB columns polled every cycle: ifTable under 1.3.6.1.2.1.2.2.1, ifXTable under 1.3.6.1.2.1.31.1.1.1.
|
||||
var (
|
||||
hcColumns = []struct {
|
||||
name, col string
|
||||
kind kind
|
||||
}{
|
||||
{"ifHCInOctets", "1.3.6.1.2.1.31.1.1.1.6", kindCounter},
|
||||
{"ifHCOutOctets", "1.3.6.1.2.1.31.1.1.1.10", kindCounter},
|
||||
}
|
||||
// Without HC counters the 32-bit octet counters are read but keep the
|
||||
// non-HC names, so a series never silently changes width under one name.
|
||||
plainColumns = []struct {
|
||||
name, col string
|
||||
kind kind
|
||||
}{
|
||||
{"ifInOctets", "1.3.6.1.2.1.2.2.1.10", kindCounter},
|
||||
{"ifOutOctets", "1.3.6.1.2.1.2.2.1.16", kindCounter},
|
||||
}
|
||||
commonColumns = []struct {
|
||||
name, col string
|
||||
kind kind
|
||||
}{
|
||||
{"ifInErrors", "1.3.6.1.2.1.2.2.1.14", kindCounter},
|
||||
{"ifOutErrors", "1.3.6.1.2.1.2.2.1.20", kindCounter},
|
||||
{"ifInDiscards", "1.3.6.1.2.1.2.2.1.13", kindCounter},
|
||||
{"ifOutDiscards", "1.3.6.1.2.1.2.2.1.19", kindCounter},
|
||||
{"ifOperStatus", "1.3.6.1.2.1.2.2.1.8", kindStatus},
|
||||
{"ifAdminStatus", "1.3.6.1.2.1.2.2.1.7", kindStatus},
|
||||
}
|
||||
)
|
||||
|
||||
func (p *Poller) buildTarget(ctx context.Context, d *store.Device) (*target, error) {
|
||||
t := &target{dev: *d}
|
||||
if a, err := netip.ParseAddr(d.MgmtIP); err == nil {
|
||||
t.addr = a
|
||||
}
|
||||
if d.SNMPCredentialID != nil {
|
||||
c, err := p.o.Store.GetCredential(ctx, *d.SNMPCredentialID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("credential: %w", err)
|
||||
}
|
||||
t.cred = c
|
||||
ifs, err := p.o.Store.ListInterfaces(ctx, d.ID, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("interfaces: %w", err)
|
||||
}
|
||||
t.oids = append(t.oids, oidSysUpTime)
|
||||
t.items = append(t.items, item{name: "sysUpTime", kind: kindUptime})
|
||||
for _, i := range ifs {
|
||||
if i.Excluded || i.RetiredAt != nil {
|
||||
continue
|
||||
}
|
||||
t.ifaces++
|
||||
octets := plainColumns
|
||||
if i.HasHCCounters {
|
||||
octets = hcColumns
|
||||
}
|
||||
for _, cols := range [][]struct {
|
||||
name, col string
|
||||
kind kind
|
||||
}{octets, commonColumns} {
|
||||
for _, c := range cols {
|
||||
t.oids = append(t.oids, fmt.Sprintf("%s.%d", c.col, i.IfIndex))
|
||||
t.items = append(t.items, item{name: c.name, iface: i.ID, kind: c.kind})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// runSNMP polls the device every interval, starting at a stable per-device offset.
|
||||
func (l *devLoop) runSNMP(ctx context.Context) {
|
||||
p := l.p
|
||||
off := offset
|
||||
if p.o.Offset != nil {
|
||||
off = p.o.Offset
|
||||
}
|
||||
next := p.clk.Now().Add(off(l.slug, p.o.Interval))
|
||||
for sleepUntil(ctx, p.clk, next) {
|
||||
l.cycle(ctx)
|
||||
next = advance(next, p.clk.Now(), p.o.Interval)
|
||||
}
|
||||
}
|
||||
|
||||
// cycle runs one poll under the cycle deadline. It returns only when the poll
|
||||
// has finished or been abandoned, so cycles of one device never overlap.
|
||||
func (l *devLoop) cycle(ctx context.Context) {
|
||||
p := l.p
|
||||
t := l.tgt.Load()
|
||||
if t == nil || t.cred == nil || !t.addr.IsValid() {
|
||||
return
|
||||
}
|
||||
start := p.clk.Now()
|
||||
cctx, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
deadline := p.clk.After(p.o.Interval / 2) // registered before polling starts
|
||||
go func() {
|
||||
select {
|
||||
case <-deadline:
|
||||
cancel(errDeadline)
|
||||
case <-cctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
samples, skipped, err := l.poll(cctx, t)
|
||||
if ctx.Err() != nil {
|
||||
return // shutting down
|
||||
}
|
||||
res := CycleResult{Device: l.slug, Err: err, Samples: len(samples), Skipped: skipped}
|
||||
if err != nil && errors.Is(context.Cause(cctx), errDeadline) {
|
||||
res.Missed = true
|
||||
l.missed.Inc()
|
||||
}
|
||||
now := p.clk.Now()
|
||||
l.duration.Set(now.Sub(start).Seconds())
|
||||
l.skipped.Add(float64(skipped))
|
||||
up := 0.0
|
||||
if err == nil {
|
||||
up = 1
|
||||
if terr := p.o.Store.TouchDevice(ctx, t.dev.ID, now.UTC()); terr != nil {
|
||||
p.log.Warn("touch device", "device", l.slug, "err", terr)
|
||||
}
|
||||
}
|
||||
samples = append(samples, metrics.New("nmon_snmp_up", up, now, metrics.LabelDevice, l.slug))
|
||||
if werr := p.o.Writer.Write(ctx, samples); werr != nil {
|
||||
p.log.Warn("write samples", "device", l.slug, "err", werr)
|
||||
}
|
||||
if l.snmpUp == nil || *l.snmpUp != (err == nil) {
|
||||
ok := err == nil
|
||||
l.snmpUp = &ok
|
||||
if ok {
|
||||
p.log.Info("snmp up", "device", l.slug)
|
||||
} else {
|
||||
p.log.Warn("snmp down", "device", l.slug, "err", err, "deadline_missed", res.Missed)
|
||||
}
|
||||
}
|
||||
if p.o.OnCycle != nil {
|
||||
p.o.OnCycle(res)
|
||||
}
|
||||
}
|
||||
|
||||
// poll does one multi-varbind GET and converts the answer to samples.
|
||||
func (l *devLoop) poll(ctx context.Context, t *target) ([]metrics.Sample, int, error) {
|
||||
p := l.p
|
||||
select {
|
||||
case p.sem <- struct{}{}:
|
||||
defer func() { <-p.sem }()
|
||||
case <-ctx.Done():
|
||||
return nil, 0, ctx.Err()
|
||||
}
|
||||
tg := discovery.Target(t.addr, p.o.SNMPPort, t.cred, p.clk)
|
||||
sess, err := p.o.Dial(ctx, tg)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer sess.Close()
|
||||
vbs, err := sess.Get(ctx, t.oids)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if len(vbs) != len(t.oids) {
|
||||
return nil, 0, fmt.Errorf("asked for %d varbinds, got %d", len(t.oids), len(vbs))
|
||||
}
|
||||
ts := p.clk.Now()
|
||||
out := make([]metrics.Sample, 0, len(vbs)+1)
|
||||
skipped := 0
|
||||
for i, vb := range vbs {
|
||||
it := t.items[i]
|
||||
if vb.Exception() || vb.OID != t.oids[i] {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
var v float64
|
||||
switch it.kind {
|
||||
case kindUptime:
|
||||
u, ok := vb.Uint64()
|
||||
if !ok {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
v = float64(u) / 100
|
||||
case kindCounter:
|
||||
u, ok := vb.Uint64()
|
||||
if !ok {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
v = float64(u)
|
||||
case kindStatus:
|
||||
n, ok := vb.Int64()
|
||||
if !ok {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
v = float64(n)
|
||||
}
|
||||
if it.iface == "" {
|
||||
out = append(out, metrics.New(it.name, v, ts, metrics.LabelDevice, l.slug))
|
||||
} else {
|
||||
out = append(out, metrics.New(it.name, v, ts, metrics.LabelDevice, l.slug, metrics.LabelIface, it.iface))
|
||||
}
|
||||
}
|
||||
return out, skipped, nil
|
||||
}
|
||||
|
||||
// runICMP probes the preferred management address every 15 s. An unsupported
|
||||
// probe writes nothing and stops: it is unknown, never down.
|
||||
func (l *devLoop) runICMP(ctx context.Context) {
|
||||
p := l.p
|
||||
next := p.clk.Now().Add(offset(l.slug, icmpInterval))
|
||||
var (
|
||||
known, up bool
|
||||
hits, misses int
|
||||
)
|
||||
for sleepUntil(ctx, p.clk, next) {
|
||||
next = advance(next, p.clk.Now(), icmpInterval)
|
||||
t := l.tgt.Load()
|
||||
if t == nil || !t.addr.IsValid() {
|
||||
continue
|
||||
}
|
||||
rtt, err := p.o.Pinger.Ping(ctx, t.addr, icmpTimeout)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if errors.Is(err, icmp.ErrUnsupported) {
|
||||
p.log.Warn("icmp probe unsupported; not probing", "device", l.slug, "err", err)
|
||||
return
|
||||
}
|
||||
now := p.clk.Now()
|
||||
var out []metrics.Sample
|
||||
if err == nil {
|
||||
hits, misses = hits+1, 0
|
||||
if !known || (!up && hits >= icmpUpAfter) {
|
||||
known, up = true, true
|
||||
}
|
||||
out = append(out, metrics.New("nmon_rtt_seconds", rtt.Seconds(), now, metrics.LabelDevice, l.slug))
|
||||
} else {
|
||||
hits, misses = 0, misses+1
|
||||
if misses >= icmpDownAfter && (!known || up) {
|
||||
known, up = true, false
|
||||
}
|
||||
}
|
||||
if known {
|
||||
v := 0.0
|
||||
if up {
|
||||
v = 1
|
||||
}
|
||||
out = append(out, metrics.New("nmon_icmp_up", v, now, metrics.LabelDevice, l.slug))
|
||||
}
|
||||
if len(out) > 0 {
|
||||
if werr := p.o.Writer.Write(ctx, out); werr != nil {
|
||||
p.log.Warn("write icmp samples", "device", l.slug, "err", werr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
276
internal/poller/poller.go
Normal file
276
internal/poller/poller.go
Normal file
@@ -0,0 +1,276 @@
|
||||
// Package poller runs the 60 s collection path: a jittered, deadline-bounded
|
||||
// SNMP GET of the generic interface counters per device, and an ICMP probe
|
||||
// every 15 s. Each device has exactly one SNMP goroutine, so a device is never
|
||||
// polled by two cycles at once; a global semaphore caps in-flight polls.
|
||||
//
|
||||
// TODO(M2): profiles instead of the hard-coded metric list, varbind
|
||||
// adaptation on tooBig, backoff and the degraded state, the 30 s cycle for
|
||||
// devices without HC counters.
|
||||
package poller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"hash/fnv"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/icmp"
|
||||
"github.com/v10networks/nmon/internal/metrics"
|
||||
"github.com/v10networks/nmon/internal/selfmon"
|
||||
"github.com/v10networks/nmon/internal/snmp"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// Defaults.
|
||||
const (
|
||||
DefaultInterval = 60 * time.Second
|
||||
DefaultMaxInflight = 512
|
||||
icmpInterval = 15 * time.Second
|
||||
selfmonInterval = 15 * time.Second
|
||||
icmpTimeout = 2 * time.Second
|
||||
icmpDownAfter = 3 // consecutive misses
|
||||
icmpUpAfter = 2 // consecutive hits after being down
|
||||
)
|
||||
|
||||
// Session is the part of *snmp.Session the poller uses.
|
||||
type Session interface {
|
||||
Get(ctx context.Context, oids []string) ([]snmp.Varbind, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Dialer opens a poll-path session. The default is snmp.Dial with path "poll".
|
||||
type Dialer func(ctx context.Context, t snmp.Target) (Session, error)
|
||||
|
||||
// Pinger sends one echo request. *icmp.Pinger implements it.
|
||||
type Pinger interface {
|
||||
Ping(ctx context.Context, addr netip.Addr, timeout time.Duration) (time.Duration, error)
|
||||
}
|
||||
|
||||
// CycleResult describes one finished SNMP cycle of one device.
|
||||
type CycleResult struct {
|
||||
Device string
|
||||
Err error // nil on success
|
||||
Missed bool // the cycle deadline passed before the poll finished
|
||||
Samples int // interface and sysUpTime samples written
|
||||
Skipped int // varbinds skipped: exceptions or non-numeric values
|
||||
}
|
||||
|
||||
// Options configures a Poller. Store and Writer are required.
|
||||
type Options struct {
|
||||
Store store.Store
|
||||
Writer metrics.Writer // the hot write queue
|
||||
Clock clock.Clock // default clock.Real()
|
||||
|
||||
Interval time.Duration // per-device SNMP interval; default 60 s. The cycle deadline is half of it
|
||||
MaxInflight int // global cap on concurrent polls; default 512
|
||||
SNMPPort int // dev/test override of UDP 161; 0 = 161
|
||||
|
||||
Registry *selfmon.Registry // default selfmon.Default
|
||||
Logger *slog.Logger // default slog.Default()
|
||||
|
||||
// Test hooks.
|
||||
Offset func(slug string, interval time.Duration) time.Duration // first-poll offset; default stable hash
|
||||
Dial Dialer
|
||||
Pinger Pinger
|
||||
OnCycle func(CycleResult)
|
||||
}
|
||||
|
||||
// Poller schedules and runs all device polls.
|
||||
type Poller struct {
|
||||
o Options
|
||||
clk clock.Clock
|
||||
log *slog.Logger
|
||||
reg *selfmon.Registry
|
||||
sem chan struct{}
|
||||
refresh chan struct{}
|
||||
started chan struct{}
|
||||
once sync.Once
|
||||
|
||||
loops map[string]*devLoop // owned by Run
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// New returns a Poller; nothing runs until Run.
|
||||
func New(o Options) *Poller {
|
||||
if o.Clock == nil {
|
||||
o.Clock = clock.Real()
|
||||
}
|
||||
if o.Interval <= 0 {
|
||||
o.Interval = DefaultInterval
|
||||
}
|
||||
if o.MaxInflight <= 0 {
|
||||
o.MaxInflight = DefaultMaxInflight
|
||||
}
|
||||
if o.Registry == nil {
|
||||
o.Registry = selfmon.Default
|
||||
}
|
||||
if o.Logger == nil {
|
||||
o.Logger = slog.Default()
|
||||
}
|
||||
if o.Dial == nil {
|
||||
o.Dial = func(ctx context.Context, t snmp.Target) (Session, error) {
|
||||
s, err := snmp.Dial(ctx, t, "poll")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
}
|
||||
if o.Pinger == nil {
|
||||
o.Pinger = icmp.New(o.Clock)
|
||||
}
|
||||
return &Poller{
|
||||
o: o, clk: o.Clock, log: o.Logger.With("component", "poller"), reg: o.Registry,
|
||||
sem: make(chan struct{}, o.MaxInflight),
|
||||
refresh: make(chan struct{}, 1),
|
||||
started: make(chan struct{}),
|
||||
loops: map[string]*devLoop{},
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh asks the poller to reload its device list from the store. It never blocks.
|
||||
func (p *Poller) Refresh() {
|
||||
select {
|
||||
case p.refresh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Started is closed once the first device list has been loaded and its loops started.
|
||||
func (p *Poller) Started() <-chan struct{} { return p.started }
|
||||
|
||||
// Run polls until ctx ends, then waits for every loop to stop.
|
||||
func (p *Poller) Run(ctx context.Context) error {
|
||||
defer p.wg.Wait()
|
||||
if err := p.reload(ctx); err != nil {
|
||||
p.log.Error("load devices", "err", err)
|
||||
}
|
||||
p.once.Do(func() { close(p.started) })
|
||||
tick := p.clk.NewTicker(selfmonInterval)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-p.refresh:
|
||||
if err := p.reload(ctx); err != nil {
|
||||
p.log.Error("reload devices", "err", err)
|
||||
}
|
||||
case <-tick.C():
|
||||
now := p.clk.Now()
|
||||
if err := p.o.Writer.Write(ctx, p.reg.Snapshot(now)); err != nil {
|
||||
p.log.Warn("write nmon_ metrics", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reload syncs the device loops with the store.
|
||||
func (p *Poller) reload(ctx context.Context) error {
|
||||
var devs []store.Device
|
||||
for off := int64(0); ; {
|
||||
page, total, err := p.o.Store.ListDevices(ctx, store.DeviceFilter{}, store.Page{Offset: off, Limit: store.MaxLimit})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
devs = append(devs, page...)
|
||||
off += int64(len(page))
|
||||
if len(page) == 0 || off >= total {
|
||||
break
|
||||
}
|
||||
}
|
||||
seen := make(map[string]bool, len(devs))
|
||||
for i := range devs {
|
||||
d := &devs[i]
|
||||
if !d.Enabled || d.RetiredAt != nil {
|
||||
continue
|
||||
}
|
||||
t, err := p.buildTarget(ctx, d)
|
||||
if err != nil {
|
||||
p.log.Error("load device", "device", d.Slug, "err", err)
|
||||
if l := p.loops[d.Slug]; l != nil {
|
||||
seen[d.Slug] = true // keep polling with the previous settings
|
||||
}
|
||||
continue
|
||||
}
|
||||
seen[d.Slug] = true
|
||||
l := p.loops[d.Slug]
|
||||
if l == nil {
|
||||
l = p.startLoop(ctx, d.Slug)
|
||||
}
|
||||
l.tgt.Store(t)
|
||||
}
|
||||
for slug, l := range p.loops {
|
||||
if !seen[slug] {
|
||||
l.cancel()
|
||||
delete(p.loops, slug)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Poller) startLoop(ctx context.Context, slug string) *devLoop {
|
||||
lctx, cancel := context.WithCancel(ctx)
|
||||
l := &devLoop{
|
||||
p: p, slug: slug, cancel: cancel,
|
||||
duration: p.reg.Gauge("nmon_poll_duration_seconds", "device", slug),
|
||||
missed: p.reg.Counter("nmon_poll_deadline_missed_total", "device", slug),
|
||||
skipped: p.reg.Counter("nmon_poll_skipped_varbinds_total", "device", slug),
|
||||
}
|
||||
p.loops[slug] = l
|
||||
p.wg.Add(2)
|
||||
go func() { defer p.wg.Done(); l.runSNMP(lctx) }()
|
||||
go func() { defer p.wg.Done(); l.runICMP(lctx) }()
|
||||
return l
|
||||
}
|
||||
|
||||
// offset spreads first polls uniformly over the interval, stably per device.
|
||||
func offset(slug string, interval time.Duration) time.Duration {
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(slug))
|
||||
return time.Millisecond + time.Duration(h.Sum64()%uint64(interval-time.Millisecond))
|
||||
}
|
||||
|
||||
// sleepUntil waits for the clock to reach t; it returns false if ctx ended first.
|
||||
func sleepUntil(ctx context.Context, clk clock.Clock, t time.Time) bool {
|
||||
d := t.Sub(clk.Now())
|
||||
if d <= 0 {
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-clk.After(d):
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// advance returns the first scheduled time after now.
|
||||
func advance(next, now time.Time, every time.Duration) time.Time {
|
||||
next = next.Add(every)
|
||||
for !next.After(now) {
|
||||
next = next.Add(every)
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
var errDeadline = errors.New("poller: cycle deadline")
|
||||
|
||||
// devLoop is the pair of goroutines serving one device.
|
||||
type devLoop struct {
|
||||
p *Poller
|
||||
slug string
|
||||
cancel context.CancelFunc
|
||||
tgt atomic.Pointer[target]
|
||||
|
||||
duration selfmon.Gauge
|
||||
missed selfmon.Counter
|
||||
skipped selfmon.Counter
|
||||
|
||||
snmpUp *bool // last state logged; touched only by the SNMP goroutine
|
||||
}
|
||||
604
internal/poller/poller_test.go
Normal file
604
internal/poller/poller_test.go
Normal file
@@ -0,0 +1,604 @@
|
||||
package poller_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gosnmp/gosnmp"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/fixturegen"
|
||||
"github.com/v10networks/nmon/internal/icmp"
|
||||
"github.com/v10networks/nmon/internal/metrics"
|
||||
"github.com/v10networks/nmon/internal/poller"
|
||||
"github.com/v10networks/nmon/internal/selfmon"
|
||||
"github.com/v10networks/nmon/internal/simagent"
|
||||
"github.com/v10networks/nmon/internal/snmp"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
"github.com/v10networks/nmon/internal/testutil"
|
||||
"github.com/v10networks/nmon/internal/ulid"
|
||||
)
|
||||
|
||||
const interval = 60 * time.Second
|
||||
|
||||
var t0 = time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
type writer struct {
|
||||
mu sync.Mutex
|
||||
all []metrics.Sample
|
||||
}
|
||||
|
||||
func (w *writer) Write(_ context.Context, s []metrics.Sample) error {
|
||||
w.mu.Lock()
|
||||
w.all = append(w.all, s...)
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *writer) samples(name string) []metrics.Sample {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
var out []metrics.Sample
|
||||
for _, s := range w.all {
|
||||
if s.Name == name {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (w *writer) snapshot() []metrics.Sample {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return append([]metrics.Sample(nil), w.all...)
|
||||
}
|
||||
|
||||
type pinger struct {
|
||||
mu sync.Mutex
|
||||
results []error // consumed in order; the last one repeats
|
||||
calls int
|
||||
}
|
||||
|
||||
func (p *pinger) Ping(context.Context, netip.Addr, time.Duration) (time.Duration, error) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var err error
|
||||
if len(p.results) > 0 {
|
||||
err = p.results[min(p.calls, len(p.results)-1)]
|
||||
}
|
||||
p.calls++
|
||||
return 3 * time.Millisecond, err
|
||||
}
|
||||
|
||||
func (p *pinger) count() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls }
|
||||
|
||||
type harness struct {
|
||||
t *testing.T
|
||||
clk *clock.Fake
|
||||
st store.Store
|
||||
w *writer
|
||||
p *poller.Poller
|
||||
cycles chan poller.CycleResult
|
||||
ping *pinger
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
counts map[string]int
|
||||
}
|
||||
|
||||
type opts struct {
|
||||
maxInflight int
|
||||
dial poller.Dialer
|
||||
offset func(string, time.Duration) time.Duration
|
||||
}
|
||||
|
||||
func newHarness(t *testing.T, clk *clock.Fake, st store.Store, o opts) *harness {
|
||||
h := &harness{t: t, clk: clk, st: st, w: &writer{}, cycles: make(chan poller.CycleResult, 4096), ping: &pinger{}, counts: map[string]int{}}
|
||||
h.p = poller.New(poller.Options{
|
||||
Store: st, Writer: h.w, Clock: clk, Interval: interval, MaxInflight: o.maxInflight,
|
||||
Dial: o.dial, Pinger: h.ping, Offset: o.offset,
|
||||
OnCycle: func(r poller.CycleResult) { h.cycles <- r },
|
||||
})
|
||||
return h
|
||||
}
|
||||
|
||||
// start runs the poller and waits until every loop is blocked on the clock.
|
||||
func (h *harness) start(devices int) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
h.cancel = cancel
|
||||
h.done = make(chan struct{})
|
||||
go func() { h.p.Run(ctx); close(h.done) }()
|
||||
h.t.Cleanup(h.stop)
|
||||
<-h.p.Started()
|
||||
// per device: SNMP and ICMP sleeps; plus the nmon_ metrics ticker
|
||||
eventually(h.t, func() bool { return h.clk.Waiters() >= 2*devices+1 })
|
||||
}
|
||||
|
||||
func (h *harness) stop() {
|
||||
h.cancel()
|
||||
<-h.done
|
||||
}
|
||||
|
||||
// drain collects finished cycles for up to wait.
|
||||
func (h *harness) drain(wait time.Duration) []poller.CycleResult {
|
||||
var out []poller.CycleResult
|
||||
for {
|
||||
select {
|
||||
case r := <-h.cycles:
|
||||
h.counts[r.Device]++
|
||||
out = append(out, r)
|
||||
case <-time.After(wait):
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// until advances the fake clock in quarter-interval steps until cond holds.
|
||||
func (h *harness) until(cond func() bool) {
|
||||
h.t.Helper()
|
||||
for i := 0; i < 40 && !cond(); i++ {
|
||||
h.clk.Advance(interval / 4)
|
||||
h.drain(25 * time.Millisecond)
|
||||
}
|
||||
if !cond() {
|
||||
h.t.Fatal("condition not reached after advancing the clock")
|
||||
}
|
||||
}
|
||||
|
||||
func eventually(t *testing.T, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for !cond() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("timed out waiting for condition")
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// fixedOffset starts every device 1 s into the interval, so all cycles share one deadline window.
|
||||
func fixedOffset(string, time.Duration) time.Duration { return time.Second }
|
||||
|
||||
func counter(name string, kv ...string) float64 {
|
||||
return selfmon.Default.Counter(name, kv...).Value()
|
||||
}
|
||||
|
||||
func TestPollWritesExactLabelsAndRisingValues(t *testing.T) {
|
||||
clk := clock.NewFake(t0)
|
||||
st := testutil.OpenStore(t, clk)
|
||||
a := testutil.StartAgent(t, simagent.Config{
|
||||
Recording: fixturegen.Generic(fixturegen.Options{Ports: 8, HCCounters: true, Counters: true}), Clock: clk,
|
||||
})
|
||||
dev := testutil.AddDevice(t, st, a, "sw1", "public", clk)
|
||||
ifs, _ := st.ListInterfaces(context.Background(), dev.ID, false)
|
||||
ids := map[string]bool{}
|
||||
for _, i := range ifs {
|
||||
ids[i.ID] = true
|
||||
}
|
||||
a.ResetStats()
|
||||
pdus := func(typ string) float64 { return counter("nmon_pdus_total", "path", "poll", "type", typ) }
|
||||
get0, next0, bulk0 := pdus("get"), pdus("getnext"), pdus("getbulk")
|
||||
|
||||
h := newHarness(t, clk, st, opts{})
|
||||
h.p = poller.New(poller.Options{
|
||||
Store: st, Writer: h.w, Clock: clk, Interval: interval, Pinger: h.ping, SNMPPort: int(a.Addr().Port()),
|
||||
OnCycle: func(r poller.CycleResult) { h.cycles <- r },
|
||||
})
|
||||
h.start(1)
|
||||
h.until(func() bool { return h.counts[dev.Slug] >= 3 })
|
||||
time.Sleep(100 * time.Millisecond) // let any cycle in flight finish; the clock no longer moves
|
||||
h.drain(10 * time.Millisecond)
|
||||
n := h.counts[dev.Slug]
|
||||
|
||||
names := map[string]bool{}
|
||||
perIface := map[string][]float64{}
|
||||
for _, s := range h.w.snapshot() {
|
||||
if len(s.Name) < 3 || s.Name[:3] == "nmo" && s.Name != "nmon_snmp_up" {
|
||||
continue // selfmon snapshots and icmp are checked elsewhere
|
||||
}
|
||||
names[s.Name] = true
|
||||
switch s.Name {
|
||||
case "sysUpTime", "nmon_snmp_up":
|
||||
if len(s.Labels) != 1 || s.Labels[0] != (metrics.Label{Name: "device", Value: dev.Slug}) {
|
||||
t.Fatalf("%s labels %v", s.Name, s.Labels)
|
||||
}
|
||||
if s.Name == "nmon_snmp_up" && s.Value != 1 {
|
||||
t.Fatalf("snmp_up %v", s.Value)
|
||||
}
|
||||
default:
|
||||
if len(s.Labels) != 2 || s.Labels[0].Name != "device" || s.Labels[0].Value != dev.Slug ||
|
||||
s.Labels[1].Name != "iface" || !ids[s.Labels[1].Value] {
|
||||
t.Fatalf("%s labels %v", s.Name, s.Labels)
|
||||
}
|
||||
if s.Name == "ifHCInOctets" {
|
||||
perIface[s.Labels[1].Value] = append(perIface[s.Labels[1].Value], s.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
want := []string{"sysUpTime", "nmon_snmp_up", "ifHCInOctets", "ifHCOutOctets", "ifInErrors", "ifOutErrors",
|
||||
"ifInDiscards", "ifOutDiscards", "ifOperStatus", "ifAdminStatus"}
|
||||
if len(names) != len(want) {
|
||||
t.Errorf("metric names %v", names)
|
||||
}
|
||||
for _, w := range want {
|
||||
if !names[w] {
|
||||
t.Errorf("missing %s", w)
|
||||
}
|
||||
}
|
||||
if got := len(h.w.samples("ifHCInOctets")); got != 8*n {
|
||||
t.Errorf("%d ifHCInOctets samples for %d cycles of 8 ports", got, n)
|
||||
}
|
||||
for id, vs := range perIface {
|
||||
for i := 1; i < len(vs); i++ {
|
||||
if vs[i] <= vs[i-1] {
|
||||
t.Errorf("iface %s: %v not rising", id, vs)
|
||||
}
|
||||
}
|
||||
}
|
||||
if up := h.w.samples("sysUpTime"); len(up) != n || up[1].Value <= up[0].Value {
|
||||
t.Errorf("sysUpTime %v", up)
|
||||
}
|
||||
|
||||
// 1 sysUpTime + 8 ports x 8 objects = 65 varbinds = 2 PDUs of at most 40, all GET.
|
||||
st1 := a.Stats()
|
||||
if st1.Get != uint64(2*n) || st1.GetNext != 0 || st1.GetBulk != 0 {
|
||||
t.Errorf("agent saw %+v for %d cycles, want %d GETs and no walks", st1, n, 2*n)
|
||||
}
|
||||
if d := pdus("get") - get0; d != float64(2*n) {
|
||||
t.Errorf("nmon_pdus_total{poll,get} rose by %v, want %d", d, 2*n)
|
||||
}
|
||||
if pdus("getnext") != next0 || pdus("getbulk") != bulk0 {
|
||||
t.Error("walk PDUs counted on the poll path")
|
||||
}
|
||||
if got, _ := st.GetDeviceBySlug(context.Background(), dev.Slug); got.LastSeen == nil {
|
||||
t.Error("last_seen not updated")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollWithoutHCUsesPlainNames(t *testing.T) {
|
||||
clk := clock.NewFake(t0)
|
||||
st := testutil.OpenStore(t, clk)
|
||||
a := testutil.StartAgent(t, simagent.Config{Recording: fixturegen.Generic(fixturegen.Options{Ports: 4, Counters: true}), Clock: clk})
|
||||
dev := testutil.AddDevice(t, st, a, "old", "public", clk)
|
||||
h := newHarness(t, clk, st, opts{})
|
||||
h.p = poller.New(poller.Options{Store: st, Writer: h.w, Clock: clk, Interval: interval, Pinger: h.ping,
|
||||
SNMPPort: int(a.Addr().Port()), OnCycle: func(r poller.CycleResult) { h.cycles <- r }})
|
||||
h.start(1)
|
||||
h.until(func() bool { return h.counts[dev.Slug] >= 1 })
|
||||
if len(h.w.samples("ifInOctets")) != 4 || len(h.w.samples("ifOutOctets")) != 4 {
|
||||
t.Errorf("want 4 ifInOctets and ifOutOctets, got %d/%d", len(h.w.samples("ifInOctets")), len(h.w.samples("ifOutOctets")))
|
||||
}
|
||||
if len(h.w.samples("ifHCInOctets")) != 0 || len(h.w.samples("ifHCOutOctets")) != 0 {
|
||||
t.Error("HC names used for a device without HC counters")
|
||||
}
|
||||
}
|
||||
|
||||
// fakeDevices creates n SNMP devices with two interfaces each, no agent behind them.
|
||||
func fakeDevices(t *testing.T, st store.Store, n int) []*store.Device {
|
||||
ctx := context.Background()
|
||||
var out []*store.Device
|
||||
for i := 0; i < n; i++ {
|
||||
name := "dev" + string(rune('a'+i))
|
||||
cred := &store.SNMPCredential{Name: name, Version: "v2c", Community: name}
|
||||
if err := st.CreateCredential(ctx, cred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &store.Device{Name: name, Kind: store.KindNetwork, SNMPCredentialID: &cred.ID, Enabled: true, Profile: "generic"}
|
||||
if err := st.CreateDevice(ctx, d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SetAddresses(ctx, d.ID, []string{"::1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ifs []store.Interface
|
||||
for j := 1; j <= 2; j++ {
|
||||
ifs = append(ifs, store.Interface{ID: ulid.New(t0), IfIndex: int64(j), IfName: "p" + string(rune('0'+j)), HasHCCounters: true})
|
||||
}
|
||||
if err := st.UpsertInterfaces(ctx, d.ID, ifs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// blockingSession answers every Get with ones after release, or fails with ctx.Err().
|
||||
type blocker struct {
|
||||
mu sync.Mutex
|
||||
cur map[string]int
|
||||
max map[string]int
|
||||
global atomic.Int32
|
||||
gmax atomic.Int32
|
||||
release chan struct{}
|
||||
dialed atomic.Int32
|
||||
}
|
||||
|
||||
func newBlocker() *blocker {
|
||||
return &blocker{cur: map[string]int{}, max: map[string]int{}, release: make(chan struct{})}
|
||||
}
|
||||
|
||||
type blockSession struct {
|
||||
b *blocker
|
||||
comm string
|
||||
}
|
||||
|
||||
func (b *blocker) dial(_ context.Context, tg snmp.Target) (poller.Session, error) {
|
||||
b.dialed.Add(1)
|
||||
return &blockSession{b, tg.Cred.Community}, nil
|
||||
}
|
||||
|
||||
func (s *blockSession) Get(ctx context.Context, oids []string) ([]snmp.Varbind, error) {
|
||||
b := s.b
|
||||
b.mu.Lock()
|
||||
b.cur[s.comm]++
|
||||
b.max[s.comm] = max(b.max[s.comm], b.cur[s.comm])
|
||||
b.mu.Unlock()
|
||||
g := b.global.Add(1)
|
||||
for {
|
||||
old := b.gmax.Load()
|
||||
if g <= old || b.gmax.CompareAndSwap(old, g) {
|
||||
break
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
b.global.Add(-1)
|
||||
b.mu.Lock()
|
||||
b.cur[s.comm]--
|
||||
b.mu.Unlock()
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-b.release:
|
||||
}
|
||||
out := make([]snmp.Varbind, len(oids))
|
||||
for i, o := range oids {
|
||||
out[i] = snmp.Varbind{OID: o, Type: gosnmp.Integer, Value: 1}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *blockSession) Close() error { return nil }
|
||||
|
||||
func TestPerDeviceConcurrencyIsOneEvenWhenDeadlineBeatsLatency(t *testing.T) {
|
||||
clk := clock.NewFake(t0)
|
||||
st := testutil.OpenStore(t, clk)
|
||||
devs := fakeDevices(t, st, 3)
|
||||
b := newBlocker()
|
||||
missed0 := map[string]float64{}
|
||||
for _, d := range devs {
|
||||
missed0[d.Slug] = counter("nmon_poll_deadline_missed_total", "device", d.Slug)
|
||||
}
|
||||
h := newHarness(t, clk, st, opts{dial: b.dial})
|
||||
h.start(3)
|
||||
h.until(func() bool {
|
||||
for _, d := range devs {
|
||||
if h.counts[d.Slug] < 3 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for _, d := range devs {
|
||||
if m := b.max[d.Name]; m != 1 {
|
||||
t.Errorf("%s: max concurrent polls %d, want 1", d.Slug, m)
|
||||
}
|
||||
got := counter("nmon_poll_deadline_missed_total", "device", d.Slug) - missed0[d.Slug]
|
||||
if got < 3 || got != float64(h.counts[d.Slug]) {
|
||||
t.Errorf("%s: deadline missed %v times in %d cycles", d.Slug, got, h.counts[d.Slug])
|
||||
}
|
||||
if dur := selfmon.Default.Gauge("nmon_poll_duration_seconds", "device", d.Slug).Value(); dur < 30 {
|
||||
t.Errorf("%s: poll duration %vs, want at least the 30s deadline", d.Slug, dur)
|
||||
}
|
||||
}
|
||||
for _, s := range h.w.samples("nmon_snmp_up") {
|
||||
if s.Value != 0 {
|
||||
t.Fatalf("snmp_up %v after a missed deadline", s.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobalInflightCap(t *testing.T) {
|
||||
clk := clock.NewFake(t0)
|
||||
st := testutil.OpenStore(t, clk)
|
||||
devs := fakeDevices(t, st, 5)
|
||||
b := newBlocker()
|
||||
h := newHarness(t, clk, st, opts{dial: b.dial, maxInflight: 2, offset: fixedOffset})
|
||||
h.start(5)
|
||||
clk.Advance(2 * time.Second) // every device starts its cycle
|
||||
eventually(t, func() bool { return b.global.Load() == 2 })
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if g := b.gmax.Load(); g != 2 {
|
||||
t.Fatalf("max in flight %d with --max-inflight 2", g)
|
||||
}
|
||||
close(b.release)
|
||||
got := map[string]bool{}
|
||||
deadline := time.After(10 * time.Second)
|
||||
for len(got) < len(devs) {
|
||||
select {
|
||||
case r := <-h.cycles:
|
||||
if r.Err != nil {
|
||||
t.Fatalf("%s: %v", r.Device, r.Err)
|
||||
}
|
||||
got[r.Device] = true
|
||||
case <-deadline:
|
||||
t.Fatalf("only %d of %d devices finished", len(got), len(devs))
|
||||
}
|
||||
}
|
||||
if g := b.gmax.Load(); g != 2 {
|
||||
t.Fatalf("max in flight %d after release", g)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlowDeviceDoesNotDelayOthers(t *testing.T) {
|
||||
clk := clock.NewFake(t0)
|
||||
st := testutil.OpenStore(t, clk)
|
||||
rec := fixturegen.Generic(fixturegen.Options{Ports: 8, HCCounters: true, Counters: true})
|
||||
ports := map[string]uint16{}
|
||||
normal := map[string]bool{}
|
||||
for i := 0; i < 10; i++ {
|
||||
name := "norm" + string(rune('a'+i))
|
||||
a := testutil.StartAgent(t, simagent.Config{Recording: rec, Clock: clk, Community: name})
|
||||
d := testutil.AddDevice(t, st, a, name, name, clk)
|
||||
ports[name] = a.Addr().Port()
|
||||
normal[d.Slug] = true
|
||||
}
|
||||
// The slow device is seeded against a fast agent, then replaced by one that answers after 3 s.
|
||||
fast := testutil.StartAgent(t, simagent.Config{Recording: rec, Clock: clk, Community: "slow"})
|
||||
slow := testutil.AddDevice(t, st, fast, "slow", "slow", clk)
|
||||
fast.Close()
|
||||
s := testutil.StartAgent(t, simagent.Config{Recording: rec, Community: "slow", Behaviour: simagent.Behaviour{Latency: 3 * time.Second}})
|
||||
ports["slow"] = s.Addr().Port()
|
||||
|
||||
var slowDialed atomic.Bool
|
||||
dial := func(ctx context.Context, tg snmp.Target) (poller.Session, error) {
|
||||
tg.Addr = netip.AddrPortFrom(tg.Addr.Addr(), ports[tg.Cred.Community])
|
||||
if tg.Cred.Community == "slow" {
|
||||
defer slowDialed.Store(true)
|
||||
}
|
||||
sess, err := snmp.Dial(ctx, tg, "poll")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return sess, nil
|
||||
}
|
||||
missed0 := counter("nmon_poll_deadline_missed_total", "device", slow.Slug)
|
||||
h := newHarness(t, clk, st, opts{dial: dial, offset: fixedOffset})
|
||||
h.start(11)
|
||||
|
||||
clk.Advance(2 * time.Second) // one tick: every device starts
|
||||
done := map[string]bool{}
|
||||
deadline := time.After(10 * time.Second)
|
||||
for len(done) < len(normal) {
|
||||
select {
|
||||
case r := <-h.cycles:
|
||||
if r.Device == slow.Slug {
|
||||
t.Fatalf("slow device finished early: %+v", r)
|
||||
}
|
||||
if r.Err != nil {
|
||||
t.Fatalf("%s: %v", r.Device, r.Err)
|
||||
}
|
||||
done[r.Device] = true
|
||||
case <-deadline:
|
||||
t.Fatalf("only %d of %d normal devices finished within the first tick", len(done), len(normal))
|
||||
}
|
||||
}
|
||||
eventually(t, slowDialed.Load)
|
||||
if got := counter("nmon_poll_deadline_missed_total", "device", slow.Slug); got != missed0 {
|
||||
t.Fatalf("deadline counted before it passed: %v", got)
|
||||
}
|
||||
clk.Advance(interval / 2) // the slow device's cycle deadline
|
||||
for {
|
||||
select {
|
||||
case r := <-h.cycles:
|
||||
if r.Device != slow.Slug {
|
||||
continue // normal devices starting their next cycle
|
||||
}
|
||||
if !r.Missed || r.Err == nil {
|
||||
t.Fatalf("slow device result %+v, want a missed deadline", r)
|
||||
}
|
||||
if got := counter("nmon_poll_deadline_missed_total", "device", slow.Slug) - missed0; got != 1 {
|
||||
t.Fatalf("deadline missed counted %v times", got)
|
||||
}
|
||||
return
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("slow device cycle never ended")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestICMPStateMachine(t *testing.T) {
|
||||
clk := clock.NewFake(t0)
|
||||
st := testutil.OpenStore(t, clk)
|
||||
d := &store.Device{Name: "vm1", Kind: store.KindVM, Enabled: true, Profile: "generic"}
|
||||
if err := st.CreateDevice(context.Background(), d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SetAddresses(context.Background(), d.ID, []string{"192.0.2.1", "2001:db8::1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
miss, hit := errors.New("timeout"), error(nil)
|
||||
h := newHarness(t, clk, st, opts{})
|
||||
h.ping.results = []error{hit, miss, miss, miss, hit, hit}
|
||||
h.start(1)
|
||||
|
||||
var ups []float64
|
||||
step := func() {
|
||||
before := h.ping.count()
|
||||
for h.ping.count() == before {
|
||||
clk.Advance(15 * time.Second)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond) // the write follows the ping
|
||||
}
|
||||
for i := 0; i < 6; i++ {
|
||||
step()
|
||||
}
|
||||
for _, s := range h.w.samples("nmon_icmp_up") {
|
||||
ups = append(ups, s.Value)
|
||||
}
|
||||
// hit: up; two misses: nothing changes; third miss: down; first hit: still down; second hit: up.
|
||||
want := []float64{1, 1, 1, 0, 0, 1}
|
||||
if len(ups) != len(want) {
|
||||
t.Fatalf("icmp_up series %v, want %v", ups, want)
|
||||
}
|
||||
for i := range want {
|
||||
if ups[i] != want[i] {
|
||||
t.Fatalf("icmp_up series %v, want %v", ups, want)
|
||||
}
|
||||
}
|
||||
if n := len(h.w.samples("nmon_rtt_seconds")); n != 3 {
|
||||
t.Errorf("%d rtt samples, want one per hit", n)
|
||||
}
|
||||
for _, s := range h.w.samples("nmon_icmp_up") {
|
||||
if len(s.Labels) != 1 || s.Labels[0].Value != d.Slug {
|
||||
t.Fatalf("labels %v", s.Labels)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestICMPUnsupportedWritesNothing(t *testing.T) {
|
||||
clk := clock.NewFake(t0)
|
||||
st := testutil.OpenStore(t, clk)
|
||||
fakeDevices(t, st, 1)
|
||||
h := newHarness(t, clk, st, opts{dial: newBlocker().dial})
|
||||
h.ping.results = []error{icmp.ErrUnsupported}
|
||||
h.start(1)
|
||||
for h.ping.count() == 0 {
|
||||
clk.Advance(15 * time.Second)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
clk.Advance(15 * time.Second)
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
if n := h.ping.count(); n != 1 {
|
||||
t.Errorf("pinged %d times after ErrUnsupported", n)
|
||||
}
|
||||
if len(h.w.samples("nmon_icmp_up")) != 0 || len(h.w.samples("nmon_rtt_seconds")) != 0 {
|
||||
t.Error("unsupported probe wrote samples")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshPicksUpNewDeviceAndSelfmonIsWritten(t *testing.T) {
|
||||
clk := clock.NewFake(t0)
|
||||
st := testutil.OpenStore(t, clk)
|
||||
b := newBlocker()
|
||||
close(b.release)
|
||||
h := newHarness(t, clk, st, opts{dial: b.dial})
|
||||
h.start(0)
|
||||
devs := fakeDevices(t, st, 1)
|
||||
h.p.Refresh()
|
||||
eventually(t, func() bool { return clk.Waiters() >= 3 })
|
||||
h.until(func() bool { return h.counts[devs[0].Slug] >= 1 })
|
||||
eventually(t, func() bool { return len(h.w.samples("nmon_poll_duration_seconds")) > 0 })
|
||||
if len(h.w.samples("nmon_pdus_total")) == 0 && len(h.w.samples("nmon_poll_duration_seconds")) == 0 {
|
||||
t.Error("selfmon snapshot not written")
|
||||
}
|
||||
}
|
||||
93
internal/testutil/testutil.go
Normal file
93
internal/testutil/testutil.go
Normal file
@@ -0,0 +1,93 @@
|
||||
// Package testutil builds the fixtures WP5 tests share: a migrated SQLite
|
||||
// store, simagent devices and inventory rows seeded by discovery.
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/discovery"
|
||||
"github.com/v10networks/nmon/internal/secrets"
|
||||
"github.com/v10networks/nmon/internal/simagent"
|
||||
"github.com/v10networks/nmon/internal/snmp"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// OpenStore returns a migrated SQLite store in a temp dir, closed at test end.
|
||||
func OpenStore(t testing.TB, clk clock.Clock) store.Store {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
key := filepath.Join(dir, "secret.key")
|
||||
if err := secrets.GenerateKeyFile(key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kr, err := secrets.LoadKeyring(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
st, err := store.Open(ctx, store.DriverSQLite, filepath.Join(dir, "inventory.db"), kr, clk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
if err := st.Migrate(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// StartAgent serves cfg on [::1] (unless cfg.Addr says otherwise) until the test ends.
|
||||
func StartAgent(t testing.TB, cfg simagent.Config) *simagent.Agent {
|
||||
t.Helper()
|
||||
a, err := simagent.Start(context.Background(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { a.Close() })
|
||||
return a
|
||||
}
|
||||
|
||||
// AddDevice creates a v2c credential with the given community and a network
|
||||
// device pointing at ::1, and seeds its interfaces from agent a.
|
||||
func AddDevice(t testing.TB, st store.Store, a *simagent.Agent, name, community string, clk clock.Clock) *store.Device {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
cred := &store.SNMPCredential{Name: name + "-cred", Version: "v2c", Community: community}
|
||||
if err := st.CreateCredential(ctx, cred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dev := &store.Device{Name: name, Kind: store.KindNetwork, SNMPCredentialID: &cred.ID, Enabled: true, Profile: "generic"}
|
||||
if err := st.CreateDevice(ctx, dev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SetAddresses(ctx, dev.ID, []string{"::1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
Seed(t, st, a, dev, cred, clk)
|
||||
got, err := st.GetDeviceByID(ctx, dev.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// Seed runs discovery.Seed for dev against agent a.
|
||||
func Seed(t testing.TB, st store.Store, a *simagent.Agent, dev *store.Device, cred *store.SNMPCredential, clk clock.Clock) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
tg := discovery.Target(a.Addr().Addr(), int(a.Addr().Port()), cred, clk)
|
||||
sess, err := snmp.Dial(ctx, tg, "discovery")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer sess.Close()
|
||||
if err := discovery.Seed(ctx, st, sess, dev, clk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
138
scripts/gate-m1.sh
Executable file
138
scripts/gate-m1.sh
Executable file
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env bash
|
||||
# Gate M1: `nmon serve` polls the counters fixture and /api/v1/query returns
|
||||
# rising ifHCInOctets. Procedure: docs/milestones/M1-skeleton-and-fixtures.md.
|
||||
# Needs: go, make, curl, file, ss, pgrep, python3 (JSON checks).
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
FAILS=0
|
||||
NMON_PID="" AGENT_PID=""
|
||||
|
||||
pass() { echo "PASS: $*"; }
|
||||
fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
|
||||
die() { fail "$*"; exit 1; }
|
||||
|
||||
cleanup() {
|
||||
local pid
|
||||
for pid in $NMON_PID $AGENT_PID; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
for pid in $NMON_PID $AGENT_PID; do
|
||||
for _ in $(seq 1 100); do kill -0 "$pid" 2>/dev/null || break; sleep 0.1; done
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
done
|
||||
if [ "$FAILS" -ne 0 ] && [ -f "$TMP/nmon.log" ]; then
|
||||
echo "--- tail of nmon.log"; tail -n 40 "$TMP/nmon.log"
|
||||
fi
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT TERM
|
||||
|
||||
pick_port() {
|
||||
local p
|
||||
while :; do
|
||||
p=$((20000 + RANDOM % 30000))
|
||||
ss -H -lntu 2>/dev/null | awk '{print $5}' | grep -q "[:.]$p\$" || { echo "$p"; return; }
|
||||
done
|
||||
}
|
||||
|
||||
command -v python3 >/dev/null || die "python3 is required for the JSON checks"
|
||||
|
||||
# 1. Build with the embedded stores; the binary must be static.
|
||||
echo "== build"
|
||||
make build >"$TMP/build.log" 2>&1 || { tail -n 20 "$TMP/build.log"; die "make build"; }
|
||||
if file nmon | grep -q 'statically linked'; then pass "nmon is statically linked"; else die "nmon is not statically linked: $(file nmon)"; fi
|
||||
|
||||
# 2. Fixture agent on [::1]:<port>.
|
||||
echo "== simagent"
|
||||
SNMP_PORT="$(pick_port)"
|
||||
go build -o "$TMP/simagent" ./internal/simagent/cmd/simagent || die "build simagent"
|
||||
"$TMP/simagent" -addr "[::1]:$SNMP_PORT" -rec testdata/snmprec/counters.snmprec >"$TMP/simagent.log" 2>&1 &
|
||||
AGENT_PID=$!
|
||||
for _ in $(seq 1 50); do grep -q listening "$TMP/simagent.log" 2>/dev/null && break; sleep 0.1; done
|
||||
grep -q listening "$TMP/simagent.log" || { cat "$TMP/simagent.log"; die "simagent did not start"; }
|
||||
pass "simagent serving counters.snmprec on [::1]:$SNMP_PORT"
|
||||
|
||||
# 3. nmon serve, data dir and key file in separate places.
|
||||
echo "== serve"
|
||||
HTTP_PORT="$(pick_port)"
|
||||
BASE="http://[::1]:$HTTP_PORT"
|
||||
./nmon serve --data-dir "$TMP/data" --secret-key-file "$TMP/keys/secret.key" --insecure-http \
|
||||
--listen "[::1]:$HTTP_PORT" --poll-interval 10s --snmp-port "$SNMP_PORT" >"$TMP/nmon.log" 2>&1 &
|
||||
NMON_PID=$!
|
||||
for _ in $(seq 1 300); do
|
||||
curl -fsS "$BASE/openapi.json" >/dev/null 2>&1 && break
|
||||
kill -0 "$NMON_PID" 2>/dev/null || break
|
||||
sleep 0.2
|
||||
done
|
||||
curl -fsS "$BASE/openapi.json" >/dev/null 2>&1 || die "nmon serve did not come up"
|
||||
pass "nmon serve is up on $BASE"
|
||||
|
||||
# 4. Add the fixture device.
|
||||
echo "== add device"
|
||||
CODE="$(curl -sS -o "$TMP/device.json" -w '%{http_code}' -X POST "$BASE/api/v1/devices" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"name":"counters","addresses":["::1"],"snmp":{"version":"v2c","community":"public"}}')"
|
||||
[ "$CODE" = 201 ] || { cat "$TMP/device.json"; die "POST /api/v1/devices returned $CODE"; }
|
||||
SLUG="$(python3 -I -c 'import json,sys; print(json.load(open(sys.argv[1]))["slug"])' "$TMP/device.json")"
|
||||
NIF="$(python3 -I -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["interfaces"]))' "$TMP/device.json")"
|
||||
[ "$NIF" -ge 1 ] || die "device $SLUG has no interfaces"
|
||||
pass "device $SLUG added with $NIF interfaces"
|
||||
|
||||
# 5. Three cycles, then query twice one cycle apart.
|
||||
echo "== query (waiting for 3 poll cycles)"
|
||||
sleep 30
|
||||
query() { curl -fsS --get "$BASE/api/v1/query" --data-urlencode 'q=ifHCInOctets' -o "$1"; }
|
||||
for _ in $(seq 1 15); do
|
||||
query "$TMP/q1.json" && python3 -I -c 'import json,sys; sys.exit(0 if json.load(open(sys.argv[1]))["data"]["result"] else 1)' "$TMP/q1.json" && break
|
||||
sleep 2
|
||||
done
|
||||
sleep 10
|
||||
|
||||
CHECK='
|
||||
import json, sys
|
||||
a = json.load(open(sys.argv[1]))["data"]["result"]
|
||||
b = {s["metric"].get("iface"): float(s["value"][1]) for s in json.load(open(sys.argv[2]))["data"]["result"]}
|
||||
if not a:
|
||||
print("no ifHCInOctets series"); sys.exit(1)
|
||||
for s in a:
|
||||
keys = sorted(s["metric"])
|
||||
if keys != ["__name__", "device", "iface"]:
|
||||
print("labels are", keys); sys.exit(1)
|
||||
i = s["metric"]["iface"]
|
||||
if i not in b or not b[i] > float(s["value"][1]):
|
||||
print("iface", i, "did not rise:", s["value"][1], "->", b.get(i)); sys.exit(1)
|
||||
print(len(a), "series, labels exactly {__name__, device, iface}, all values strictly rising")
|
||||
'
|
||||
OK=0
|
||||
for _ in $(seq 1 6); do
|
||||
query "$TMP/q2.json" && OUT="$(python3 -I -c "$CHECK" "$TMP/q1.json" "$TMP/q2.json" 2>&1)" && { OK=1; break; }
|
||||
sleep 2
|
||||
done
|
||||
if [ "$OK" = 1 ]; then pass "ifHCInOctets: $OUT"; else fail "ifHCInOctets check: ${OUT:-query failed}"; fi
|
||||
|
||||
# 6. The metric stores listen on loopback only.
|
||||
echo "== listeners"
|
||||
PIDS="$(pgrep -P "$NMON_PID" | tr '\n' ' ')"
|
||||
N=0 BAD=0
|
||||
while read -r addr pid; do
|
||||
N=$((N + 1))
|
||||
case "$addr" in 127.0.0.1:*) ;; *) BAD=$((BAD + 1)); echo " store pid $pid listens on $addr" ;; esac
|
||||
done < <(ss -H -ltnp | awk '{
|
||||
n = split($0, parts, "pid="); for (i = 2; i <= n; i++) { split(parts[i], p, ","); print $4, p[1] } }' |
|
||||
while read -r addr pid; do for c in $PIDS; do [ "$c" = "$pid" ] && echo "$addr $pid"; done; done)
|
||||
if [ "$N" -ge 2 ] && [ "$BAD" -eq 0 ]; then pass "$N store listeners, all on 127.0.0.1"
|
||||
elif [ "$N" -lt 2 ]; then fail "found $N store listeners (children: ${PIDS:-none}), expected hot and long"
|
||||
else fail "$BAD store listeners are not on 127.0.0.1"; fi
|
||||
|
||||
# 7. Graceful shutdown.
|
||||
echo "== shutdown"
|
||||
kill -TERM "$NMON_PID"
|
||||
if wait "$NMON_PID"; then pass "nmon exited cleanly on SIGTERM"; else fail "nmon exited with status $?"; fi
|
||||
NMON_PID=""
|
||||
|
||||
if [ "$FAILS" -eq 0 ]; then echo "GATE M1: PASS"; else echo "GATE M1: FAIL ($FAILS)"; exit 1; fi
|
||||
Reference in New Issue
Block a user