Compare commits
1 Commits
m6-integra
...
m5
| Author | SHA1 | Date | |
|---|---|---|---|
| 4a57eac1ed |
@@ -19,7 +19,7 @@ import (
|
||||
)
|
||||
|
||||
var errUsage = errors.New("usage: nmon-devtools backfill|seed-demo|topology-agents|topology-add|webhook-sink|gate-dump|" +
|
||||
"proxmox-mock|push-replay|backfill-long|oidc-mock|bcrypt [flags]" +
|
||||
"proxmox-mock|push-replay|backfill-long [flags]" +
|
||||
" (see docs/dev.md)")
|
||||
|
||||
func main() {
|
||||
@@ -54,10 +54,6 @@ func run(ctx context.Context, args []string) error {
|
||||
return pushReplayCmd(ctx, args[1:])
|
||||
case "backfill-long":
|
||||
return backfillLongCmd(ctx, args[1:])
|
||||
case "bcrypt":
|
||||
return bcryptCmd(ctx, args[1:])
|
||||
case "oidc-mock":
|
||||
return oidcMockCmd(ctx, args[1:])
|
||||
default:
|
||||
return fmt.Errorf("unknown command %q: %w", args[0], errUsage)
|
||||
}
|
||||
@@ -92,7 +88,7 @@ func devFlagSet(name string) *flag.FlagSet {
|
||||
return fs
|
||||
}
|
||||
|
||||
// httpJSON sends an optional JSON body (with the API token in NMON_TOKEN, if set) and decodes a 2xx reply into out (nil: discard).
|
||||
// httpJSON sends an optional JSON body and decodes a 2xx JSON reply into out (nil to discard it).
|
||||
func httpJSON(ctx context.Context, hc *http.Client, method, url string, in, out any) error {
|
||||
var body io.Reader
|
||||
if in != nil {
|
||||
@@ -109,9 +105,6 @@ func httpJSON(ctx context.Context, hc *http.Client, method, url string, in, out
|
||||
if in != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if tok := os.Getenv("NMON_TOKEN"); tok != "" { // an nmon with authentication on needs an admin API token
|
||||
req.Header.Set("Authorization", "Bearer "+tok)
|
||||
}
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/v10networks/nmon/internal/auth/oidctest"
|
||||
)
|
||||
|
||||
// devTB lets the test-only mock issuer run outside a test binary: it supplies the three methods oidctest uses
|
||||
// (Helper, Fatal, Cleanup) and leaves the rest of testing.TB nil.
|
||||
type devTB struct {
|
||||
testing.TB
|
||||
cleanups []func()
|
||||
}
|
||||
|
||||
func (*devTB) Helper() {}
|
||||
|
||||
func (*devTB) Fatal(args ...any) { panic(fmt.Sprint(args...)) }
|
||||
|
||||
func (d *devTB) Cleanup(f func()) { d.cleanups = append(d.cleanups, f) }
|
||||
|
||||
func (d *devTB) close() {
|
||||
for i := len(d.cleanups) - 1; i >= 0; i-- {
|
||||
d.cleanups[i]()
|
||||
}
|
||||
}
|
||||
|
||||
// oidcMockCmd runs internal/auth/oidctest as a standalone OpenID Connect issuer for the M6 gate and for
|
||||
// screenshots. The issuer approves every login at once as the "current user"; a control listener changes who
|
||||
// that is: GET /user?sub=ID&name=NAME&groups=a,b on --control. The issuer URL is written to --issuer-file.
|
||||
func oidcMockCmd(ctx context.Context, args []string) error {
|
||||
fs := devFlagSet("oidc-mock")
|
||||
control := fs.String("control", "[::1]:0", "listen address of the control endpoint")
|
||||
clientID := fs.String("client-id", "nmon", "OIDC client id the issuer accepts")
|
||||
clientSecret := fs.String("client-secret", "s3cret", "OIDC client secret the issuer accepts")
|
||||
groupsClaim := fs.String("groups-claim", "groups", "name of the groups claim")
|
||||
issuerFile := fs.String("issuer-file", "", "file that receives the issuer URL")
|
||||
portFile := fs.String("port-file", "", "file that receives the control port")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return flagErr(err)
|
||||
}
|
||||
tb := &devTB{}
|
||||
defer tb.close()
|
||||
srv := oidctest.New(tb)
|
||||
srv.ClientID, srv.ClientSecret, srv.GroupsClaim = *clientID, *clientSecret, *groupsClaim
|
||||
ln, err := net.Listen("tcp", *control)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writePort(*portFile, ln); err != nil {
|
||||
return err
|
||||
}
|
||||
if *issuerFile != "" {
|
||||
if err := os.WriteFile(*issuerFile, []byte(srv.Issuer()), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /user", func(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
u := oidctest.User{Sub: q.Get("sub"), Name: q.Get("name"), Email: q.Get("email")}
|
||||
if g := q.Get("groups"); g != "" {
|
||||
u.Groups = strings.Split(g, ",")
|
||||
}
|
||||
if u.Sub == "" {
|
||||
http.Error(w, "sub is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
srv.SetUser(u)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
})
|
||||
hs := &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second}
|
||||
go func() { <-ctx.Done(); _ = hs.Close() }()
|
||||
fmt.Printf("oidc-mock issuer %s control %s\n", srv.Issuer(), ln.Addr())
|
||||
if err := hs.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bcryptCmd prints the bcrypt hash of its argument: the content of a break-glass password file.
|
||||
func bcryptCmd(_ context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("usage: nmon-devtools bcrypt PASSWORD")
|
||||
}
|
||||
h, err := bcrypt.GenerateFromPassword([]byte(args[0]), bcrypt.DefaultCost)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(h))
|
||||
return nil
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// apiFlags are the flags of every command that talks to a running nmon.
|
||||
type apiFlags struct {
|
||||
URL, Token string
|
||||
SkipVerify bool
|
||||
}
|
||||
|
||||
func (f *apiFlags) register(fs *flag.FlagSet) {
|
||||
fs.StringVar(&f.URL, "api", os.Getenv("NMON_API"), "base URL of the running nmon, such as https://nmon.example.net:8443 (or set NMON_API)")
|
||||
fs.StringVar(&f.Token, "token", os.Getenv("NMON_TOKEN"), "admin API token, nma_... (or set NMON_TOKEN)")
|
||||
fs.BoolVar(&f.SkipVerify, "tls-skip-verify", false,
|
||||
"do not verify the server certificate (a self-signed certificate on a trusted network)")
|
||||
}
|
||||
|
||||
func (f *apiFlags) validate() error {
|
||||
if f.URL == "" {
|
||||
return errors.New("--api is required (the URL of the running nmon)")
|
||||
}
|
||||
if f.Token == "" {
|
||||
return errors.New("--token is required (an admin API token; create one in Settings, or POST /api/v1/tokens)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// client returns an HTTP client without an overall timeout: a backup download can take minutes. ctx bounds each call.
|
||||
func (f *apiFlags) client() *http.Client {
|
||||
tr := http.DefaultTransport.(*http.Transport).Clone()
|
||||
if f.SkipVerify {
|
||||
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12} // opt-in flag for self-signed certificates
|
||||
}
|
||||
return &http.Client{Transport: tr, Timeout: 0}
|
||||
}
|
||||
|
||||
// call sends one request and returns the open response when the status is 2xx; any other status is an error
|
||||
// carrying the problem detail. The caller closes the body.
|
||||
func (f *apiFlags) call(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
||||
req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(f.URL, "/")+path, body)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+f.Token)
|
||||
resp, err := f.client().Do(req)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
defer cancel()
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
return nil, problemError(resp)
|
||||
}
|
||||
resp.Body = &cancelOnClose{ReadCloser: resp.Body, cancel: cancel}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
type cancelOnClose struct {
|
||||
io.ReadCloser
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (c *cancelOnClose) Close() error { defer c.cancel(); return c.ReadCloser.Close() }
|
||||
|
||||
// problemError turns an RFC 9457 response into an error with the server's explanation.
|
||||
func problemError(resp *http.Response) error {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||
var p struct {
|
||||
Title, Detail string
|
||||
}
|
||||
if json.Unmarshal(b, &p) == nil && (p.Title != "" || p.Detail != "") {
|
||||
return fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(p.Title+": "+p.Detail))
|
||||
}
|
||||
return fmt.Errorf("%s", resp.Status)
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/v10networks/nmon/internal/backup"
|
||||
)
|
||||
|
||||
func init() {
|
||||
register("backup", "write a backup of inventory, metrics, config and profiles from a running nmon", backupCmd)
|
||||
}
|
||||
|
||||
// backupCmd asks the running nmon to take its online backup and saves the stream. A running nmon owns the
|
||||
// stores, so the work happens inside it (POST /api/v1/admin/backup); this command only downloads the result.
|
||||
func backupCmd(ctx context.Context, args []string) error {
|
||||
var af apiFlags
|
||||
var includeKey bool
|
||||
fs := newFlagSet("backup", "backup FILE --api URL --token TOKEN [--include-key]")
|
||||
af.register(fs)
|
||||
fs.BoolVar(&includeKey, "include-key", false, "put the encryption key file in the archive (protect the archive like the key)")
|
||||
file, rest, err := leadingArg(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fs.Parse(rest); err != nil {
|
||||
return flagErr(err)
|
||||
}
|
||||
if file == "" && fs.NArg() == 1 {
|
||||
file = fs.Arg(0)
|
||||
}
|
||||
if file == "" || fs.NArg() > 1 {
|
||||
return errors.New("usage: nmon backup FILE --api URL --token TOKEN [--include-key]")
|
||||
}
|
||||
if err := af.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := af.call(ctx, "POST", "/api/v1/admin/backup?"+url.Values{"include_key": {strconv.FormatBool(includeKey)}}.Encode(), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
included := resp.Header.Get("X-Nmon-Key-Included") == "true"
|
||||
n, err := saveAtomically(file, resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: %w", err)
|
||||
}
|
||||
fmt.Println(backup.KeyNotice(included))
|
||||
fmt.Printf("backup: wrote %s (%d bytes)\n", file, n)
|
||||
return nil
|
||||
}
|
||||
|
||||
// leadingArg splits a positional first argument from the flags after it ("nmon backup FILE --api ...").
|
||||
func leadingArg(args []string) (string, []string, error) {
|
||||
if len(args) > 0 && len(args[0]) > 0 && args[0][0] != '-' {
|
||||
return args[0], args[1:], nil
|
||||
}
|
||||
return "", args, nil
|
||||
}
|
||||
|
||||
// saveAtomically writes r to a temp file beside path, syncs it and renames it, so an interrupted download never
|
||||
// leaves a truncated file under the final name.
|
||||
func saveAtomically(path string, r io.Reader) (n int64, err error) {
|
||||
f, err := os.CreateTemp(filepath.Dir(path), ".nmon-backup-*")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = os.Remove(f.Name()) // best effort: the download error is the one to report
|
||||
}
|
||||
}()
|
||||
if n, err = io.Copy(f, r); err != nil {
|
||||
_ = f.Close() // best effort: the copy error is the one to report
|
||||
return 0, err
|
||||
}
|
||||
if err = f.Sync(); err != nil {
|
||||
_ = f.Close() // best effort: the sync error is the one to report
|
||||
return 0, err
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err = os.Chmod(f.Name(), 0o600); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, os.Rename(f.Name(), path)
|
||||
}
|
||||
@@ -31,6 +31,8 @@ func init() {
|
||||
for name, summary := range map[string]string{
|
||||
"discover": "run discovery for a device now",
|
||||
"profile": "test, record and draft device profiles",
|
||||
"backup": "write a backup of inventory, metrics, config and profiles",
|
||||
"restore": "restore a backup into a clean data dir",
|
||||
"loadtest": "run the scale test against simulated devices",
|
||||
} {
|
||||
if _, ok := commands[name]; !ok {
|
||||
|
||||
229
cmd/nmon/ops.go
@@ -1,229 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/v10networks/nmon/internal/app"
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
"github.com/v10networks/nmon/internal/keyrotate"
|
||||
"github.com/v10networks/nmon/internal/secrets"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
func init() {
|
||||
register("key", "rotate the encryption key: nmon key rotate (nmon stopped)", keyCmd)
|
||||
register("unlock", "clear break-glass lockouts: nmon unlock [IP|all]", unlockCmd)
|
||||
register("secret", "store a named secret before the first start: nmon secret set NAME", secretCmd)
|
||||
register("store", "operate the metric stores of a running nmon: nmon store restart [hot|long]", storeCmd)
|
||||
}
|
||||
|
||||
// openInventory opens the inventory named by the shared store flags. keys may be nil for commands that never
|
||||
// decrypt (unlock).
|
||||
func openInventory(ctx context.Context, f *config.Flags, kr *secrets.Keyring) (store.Store, error) {
|
||||
if f.DataDir == "" {
|
||||
return nil, errors.New("--data-dir is required")
|
||||
}
|
||||
dsn := f.DBDSN
|
||||
if dsn == "" {
|
||||
if f.DBDriver != store.DriverSQLite {
|
||||
return nil, fmt.Errorf("--db-dsn is required for --db-driver %s", f.DBDriver)
|
||||
}
|
||||
dsn = config.InventoryDB(f.DataDir)
|
||||
}
|
||||
return store.Open(ctx, f.DBDriver, dsn, kr, clock.Real())
|
||||
}
|
||||
|
||||
// keyCmd is "nmon key rotate": a new key version for new data, every credential re-sealed under it, the old
|
||||
// key kept in the file so older backups still decrypt. nmon must be stopped: a running process holds the old keys.
|
||||
func keyCmd(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 || args[0] != "rotate" {
|
||||
return errors.New("usage: nmon key rotate --data-dir DIR --secret-key-file FILE (stop nmon first)")
|
||||
}
|
||||
var f config.Flags
|
||||
fs := newFlagSet("key rotate", "key rotate --data-dir DIR --secret-key-file FILE [flags]")
|
||||
storeFlags(fs, &f)
|
||||
fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile, "the key file to rotate")
|
||||
if err := fs.Parse(args[1:]); err != nil {
|
||||
return flagErr(err)
|
||||
}
|
||||
res, err := keyrotate.Rotate(ctx, f.SecretKeyFile, func(ctx context.Context, kr *secrets.Keyring) (store.Store, error) {
|
||||
st, err := openInventory(ctx, &f, kr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return st, st.Migrate(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("key rotate: key version %d -> %d; re-encrypted %d SNMP credentials and %d secrets; "+
|
||||
"the previous key stays in %s so older backups still restore\n",
|
||||
res.OldVersion, res.NewVersion, res.Credentials, res.Secrets, f.SecretKeyFile)
|
||||
return nil
|
||||
}
|
||||
|
||||
// unlockCmd clears the break-glass lockout of one address, or of all. It works on the database, so it takes
|
||||
// effect in a running nmon too.
|
||||
func unlockCmd(ctx context.Context, args []string) error {
|
||||
target, rest, err := leadingArg(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var f config.Flags
|
||||
fs := newFlagSet("unlock", "unlock [IP|all] --data-dir DIR [flags]")
|
||||
storeFlags(fs, &f)
|
||||
if err := fs.Parse(rest); err != nil {
|
||||
return flagErr(err)
|
||||
}
|
||||
if target == "" && fs.NArg() == 1 {
|
||||
target = fs.Arg(0)
|
||||
}
|
||||
if target == "" {
|
||||
target = "all"
|
||||
}
|
||||
st, err := openInventory(ctx, &f, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = st.Close() }() // the unlock is committed
|
||||
if err := st.Migrate(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := auth.Unlock(ctx, st, "console", target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("unlock: cleared %d lockout(s) for %s\n", n, target)
|
||||
return nil
|
||||
}
|
||||
|
||||
// storeCmd is "nmon store restart [hot|long]": resume a store the supervisor gave up on.
|
||||
func storeCmd(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 || args[0] != "restart" {
|
||||
return errors.New("usage: nmon store restart [hot|long] --api URL --token TOKEN")
|
||||
}
|
||||
name, rest, err := leadingArg(args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var af apiFlags
|
||||
fs := flag.NewFlagSet("store restart", flag.ContinueOnError)
|
||||
af.register(fs)
|
||||
if err := fs.Parse(rest); err != nil {
|
||||
return flagErr(err)
|
||||
}
|
||||
if err := af.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
names := []string{"hot", "long"}
|
||||
if name != "" {
|
||||
names = []string{name}
|
||||
}
|
||||
var errs []error
|
||||
for _, n := range names {
|
||||
resp, err := af.call(ctx, http.MethodPost, "/api/v1/admin/stores/"+n+"/restart", nil)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: %w", n, err))
|
||||
continue
|
||||
}
|
||||
_ = resp.Body.Close() // 204: nothing to read
|
||||
fmt.Printf("store restart: %s store resumed\n", n)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// secretCmd is "nmon secret set NAME": it stores a named secret (the OIDC client secret, a Proxmox token) in the
|
||||
// inventory without a running nmon, reading the value from --value-file or standard input. It exists because
|
||||
// OIDC needs its client secret at start while the API that stores secrets needs a way to sign in.
|
||||
func secretCmd(ctx context.Context, args []string) error {
|
||||
if len(args) < 2 || args[0] != "set" {
|
||||
return errors.New("usage: nmon secret set NAME --data-dir DIR --secret-key-file FILE [--value-file FILE] (value on stdin otherwise)")
|
||||
}
|
||||
name, rest, err := leadingArg(args[1:])
|
||||
if err != nil || name == "" {
|
||||
return errors.New("usage: nmon secret set NAME --data-dir DIR --secret-key-file FILE")
|
||||
}
|
||||
var f config.Flags
|
||||
var valueFile string
|
||||
fs := newFlagSet("secret set", "secret set NAME --data-dir DIR --secret-key-file FILE [flags]")
|
||||
storeFlags(fs, &f)
|
||||
fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile, "the encryption key file")
|
||||
fs.StringVar(&valueFile, "value-file", "", "read the secret from this file instead of standard input")
|
||||
if err := fs.Parse(rest); err != nil {
|
||||
return flagErr(err)
|
||||
}
|
||||
value, err := readSecretValue(valueFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kr, err := keyringForSecrets(&f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
st, err := openInventory(ctx, &f, kr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = st.Close() }() // the secret is committed
|
||||
if err := st.Migrate(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := st.PutSecret(ctx, name, value); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := st.AppendAudit(ctx, "console", "secret.put", "secret:"+name, nil, nil); err != nil { // the value is never audited
|
||||
return err
|
||||
}
|
||||
fmt.Printf("secret set: stored %q (encrypted with key version %d)\n", name, kr.CurrentVersion())
|
||||
return nil
|
||||
}
|
||||
|
||||
// readSecretValue reads the secret from a file or standard input, without its trailing newline.
|
||||
func readSecretValue(valueFile string) (string, error) {
|
||||
var src io.Reader = os.Stdin
|
||||
if valueFile != "" {
|
||||
vf, err := os.Open(valueFile)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = vf.Close() }() // read-only
|
||||
src = vf
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(src, 4097))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
value := strings.TrimRight(string(raw), "\r\n")
|
||||
if value == "" || len(value) > 4096 {
|
||||
return "", errors.New("the secret must be 1 to 4096 bytes")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// keyringForSecrets loads the key file. On a brand-new install (no key file and no inventory yet) it creates the
|
||||
// key first, as `nmon serve` would, so the OIDC client secret can be stored before the first start.
|
||||
func keyringForSecrets(f *config.Flags) (*secrets.Keyring, error) {
|
||||
if _, err := os.Stat(f.SecretKeyFile); errors.Is(err, os.ErrNotExist) {
|
||||
if _, err := os.Stat(config.InventoryDB(f.DataDir)); err == nil {
|
||||
return nil, fmt.Errorf("secret key file %s is missing but the inventory exists; restore the key file", f.SecretKeyFile)
|
||||
}
|
||||
if err := secrets.GenerateKeyFile(f.SecretKeyFile); err != nil {
|
||||
return nil, fmt.Errorf("generate secret key: %w", err)
|
||||
}
|
||||
fmt.Printf("secret set: created a new encryption key at %s; back it up now\n", f.SecretKeyFile)
|
||||
}
|
||||
kr, err := secrets.LoadKeyring(f.SecretKeyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("secret key: %w", err)
|
||||
}
|
||||
return kr, nil
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/v10networks/nmon/internal/app"
|
||||
"github.com/v10networks/nmon/internal/backup"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
"github.com/v10networks/nmon/internal/vmsup"
|
||||
)
|
||||
|
||||
func init() {
|
||||
register("restore", "restore a backup into a clean data dir (nmon stopped)", restoreCmd)
|
||||
}
|
||||
|
||||
// restoreCmd unpacks a backup into an empty data directory and verifies it before it reports success. It
|
||||
// runs with nmon stopped, because it starts the restored metric stores itself to compare their series.
|
||||
func restoreCmd(ctx context.Context, args []string) error {
|
||||
var f config.Flags
|
||||
fs := newFlagSet("restore", "restore FILE --data-dir CLEAN_DIR [--secret-key-file PATH]")
|
||||
fs.StringVar(&f.DataDir, "data-dir", "", "the clean (absent or empty) data folder to restore into (required)")
|
||||
fs.StringVar(&f.SecretKeyFile, "secret-key-file", "",
|
||||
"where to write the key file if the backup contains one (default <data-dir>/secret.key; nmon serve wants it outside the data dir)")
|
||||
file, rest, err := leadingArg(args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fs.Parse(rest); err != nil {
|
||||
return flagErr(err)
|
||||
}
|
||||
if file == "" && fs.NArg() == 1 {
|
||||
file = fs.Arg(0)
|
||||
}
|
||||
if file == "" || f.DataDir == "" || fs.NArg() > 1 {
|
||||
return errors.New("usage: nmon restore FILE --data-dir CLEAN_DIR")
|
||||
}
|
||||
res, err := backup.Restore(ctx, backup.RestoreOptions{
|
||||
File: file, DataDir: f.DataDir, KeyFile: f.SecretKeyFile, Log: os.Stdout,
|
||||
Verifier: backup.NewVMVerifier(vmsup.Options{DataDir: f.DataDir}),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("restore: done and verified: %d inventory rows and %d series matched the manifest\n", res.RowsVerified, res.SeriesVerified)
|
||||
if res.KeyRestoredTo != "" {
|
||||
fmt.Printf("restore: the encryption key was restored to %s\n", res.KeyRestoredTo)
|
||||
} else {
|
||||
fmt.Printf("restore: this backup has NO key file; start nmon with --secret-key-file pointing at the key that sealed it "+
|
||||
"(default %s)\n", app.DefaultSecretKeyFile)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -51,22 +51,12 @@ func storeFlags(fs *flag.FlagSet, f *config.Flags) {
|
||||
|
||||
func serve(ctx context.Context, args []string) error {
|
||||
var f config.Flags
|
||||
fs := newFlagSet("serve", "serve --data-dir DIR (--tls-cert F --tls-key F | --acme-host H | --insecure-http) [flags]")
|
||||
fs := newFlagSet("serve", "serve --data-dir DIR --insecure-http [flags]")
|
||||
storeFlags(fs, &f)
|
||||
fs.StringVar(&f.SecretKeyFile, "secret-key-file", app.DefaultSecretKeyFile,
|
||||
"SNMP credential key file; created on first run; must be outside --data-dir")
|
||||
fs.StringVar(&f.Listen, "listen", app.DefaultListen, "address for the API")
|
||||
fs.BoolVar(&f.InsecureHTTP, "insecure-http", false,
|
||||
"development only: serve plain HTTP (insecure cookies); with no auth.oidc or auth.break_glass_file configured, "+
|
||||
"authentication is off and every request is an admin")
|
||||
fs.StringVar(&f.TLSCert, "tls-cert", "", "PEM certificate chain for HTTPS (reloaded on SIGHUP); needs --tls-key")
|
||||
fs.StringVar(&f.TLSKey, "tls-key", "", "PEM private key for --tls-cert")
|
||||
fs.Func("acme-host", "obtain a Let's Encrypt certificate for this host name (repeatable); needs port 80 reachable", func(v string) error {
|
||||
f.ACMEHosts = append(f.ACMEHosts, v)
|
||||
return nil
|
||||
})
|
||||
fs.StringVar(&f.ACMEEmail, "acme-email", "", "contact address for the ACME account")
|
||||
fs.StringVar(&f.ACMEListen, "acme-listen", app.DefaultACMEListen, "address of the ACME http-01 challenge listener")
|
||||
fs.BoolVar(&f.InsecureHTTP, "insecure-http", false, "serve plain HTTP with no authentication (required until TLS and auth arrive in M6)")
|
||||
fs.StringVar(&f.VMHotURL, "vm-hot-url", "", "use this VictoriaMetrics instance as the hot store instead of the embedded one")
|
||||
fs.StringVar(&f.VMLongURL, "vm-long-url", "", "use this VictoriaMetrics instance as the long store instead of the embedded one")
|
||||
fs.StringVar(&f.TrapListen, "trap-listen", app.DefaultTrapListen,
|
||||
|
||||
@@ -35,10 +35,3 @@ conscious, reviewed exception.
|
||||
| 28 | Pushed series carry only `device` | Pushed series also carry `push="1"` (senders cannot set it). | The device page lists what a push sender wrote without guessing from metric names. | M5 |
|
||||
| 29 | Ingest caps, late p95 months and a stalled rollup "raise an alert" | They are rows of the alerts table with rules `ingest_<kind>`, `p95_late`, `rollup_stalled` and no device, kept open until cleared (ingest alarms close 20 min after the last trip). | They are not metric rules and have no device; the metric-rule evaluator leaves these rule names alone. | M5 |
|
||||
| 30 | Proxmox cluster config could carry the token secret | Only `secret_name` exists; the secret is stored encrypted via `PUT /api/v1/secrets/{name}` and cannot be read back. | `config.yaml` is plain text. | M5 |
|
||||
| 31 | HTTPS by default; plain HTTP only with `--insecure-http` (Auth: OIDC only) | With `--insecure-http` **and** neither `auth.oidc` nor `auth.break_glass_file` configured, nmon runs in **dev-open mode**: every request is the admin principal `dev`, a loud warning is logged at start and the UI shows "Authentication disabled — development mode". Any start over TLS with no sign-in method configured refuses to start. If either sign-in method is configured, `--insecure-http` only means plain HTTP and cookies without `Secure`; authentication stays on. | The M1-M5 gates (and local development) run `nmon serve --insecure-http` with no IdP; M6 must not break them. The refusal for TLS-without-auth keeps the unsafe mode from ever being a production start. | M6 |
|
||||
| 32 | Settings (retention, alert thresholds, IdP, receivers, outbound allow-list) are edited in the UI and reloaded | `PUT /api/v1/settings` and SIGHUP reload in place: alert receivers, the outbound allow-list, trusted proxies, the nightly backup schedule, disk thresholds and the retired-device purge period. Alert rules and their interval, retention, timezone and sites, Proxmox, checks, `auth.oidc`, break-glass, session lifetimes, listen address, TLS and the data dir are saved but reported as `restart_required`. An invalid document is a 422 and changes nothing; the file is rewritten from the validated structure, so comments in a hand-edited `config.yaml` are lost by a UI save. | Those parts are built once at start (stores, evaluators, the OIDC client); rebuilding them live is a larger change than the risk of a restart warrants for v1. | M6 |
|
||||
| 33 | `nmon backup <file>` takes an online snapshot | `nmon backup` is a client of the running nmon: it calls `POST /api/v1/admin/backup` (admin token), nmon runs the backup in a scratch file under `<data-dir>/backups/.downloads/`, streams it and deletes it, and the client saves it atomically (temp file + rename). The key-included line is printed from the archive's own manifest. `nmon restore` runs offline and starts the restored stores for a moment to verify series. | Only the running process may touch the live stores and the database. | M6 |
|
||||
| 34 | (not specified) | Beyond the spec, additive to the API/CLI: `GET /api/v1/me`, `GET /api/v1/auth/options` (public), `GET /api/v1/audit` (admin), `POST /api/v1/admin/backup`, `POST /api/v1/admin/stores/{name}/restart`, `POST /api/v1/devices/{slug}/purge`, `POST /api/v1/devices/import`, `GET/PUT /api/v1/settings`; `nmon secret set NAME` (stores the OIDC client secret before the first start; it also creates the key on a brand-new install); API tokens (viewer, admin) share `/api/v1/tokens` with push tokens and have ids like `a7` because the two kinds live in different tables. | OIDC needs its client secret at start, while the API that stores secrets needs a way to sign in: the CLI breaks the cycle. The audit list makes "every use is logged" checkable. | M6 |
|
||||
| 35 | Port descriptions imported from CSV | Interfaces have a new operator-owned `description` column (migration 8). The CSV import and the API write it; discovery rewrites `ifalias` from the device on every run and never touches `description`. Interface search `q` matches ifname, ifalias and description. | A description typed into `ifalias` would be overwritten at the next discovery. | M6 |
|
||||
| 36 | OIDC login with a browser flow; break-glass as a local login | **No login UI** (owner decision). Every route except `/auth/*`, `/healthz` and push ingest needs a credential, the SPA's files included: an unauthenticated page load is a 302 to `/auth/login?return=<url>` (the OIDC flow; the return target must be a same-site path), an API call without credentials is a 401 problem+json. Break-glass is `GET /auth/breakglass` with HTTP Basic, so the browser's own prompt appears; success sets a session and redirects to `return` or `/` (`POST /auth/breakglass` with JSON stays for scripts). When no IdP is configured the redirect goes to the break-glass prompt. Logout is `POST /auth/logout`, then `/auth/signedout` (static HTML). `/docs` and `/openapi*` need the viewer role. A missing OIDC client secret at start leaves OIDC off with a warning. | The IdP is the login page; a second one would be a second thing to secure. | M6 |
|
||||
| 37 | Disk guard: "ingest throttled below the reserve" | Throttle answers push ingest with 429 + Retry-After below the 5% reserve; the poller keeps writing (its volume is bounded by the inventory). Above 90% used, new series are dropped on the nmon side (poller, push and checks) while known series continue; disk alarms are re-raised each minute while the condition holds and resolve themselves 3 minutes after it clears. | VictoriaMetrics series limits are start-up flags; the guard is nmon-side admission. | M6 |
|
||||
|
||||
@@ -44,16 +44,7 @@ by the store.
|
||||
|
||||
It starts the simulated SNMP agents in-process on `[::1]`, each on its own port (passed to the API as `snmp.port`), adds the devices through the API (a 48-port switch, access switches, routers, a firewall, a UPS, the counters fixture, a v1 device with 32-bit counters, and one whose agent is stopped after seeding so it reads as down), waits for live polls, back-fills 31 days, writes `NMON_DEVICE`, `NMON_DEVICE_48`, `NMON_DEVICE_32BIT` and `NMON_DEVICE_DOWN` to `--env-out`, and keeps the agents answering until interrupted. The simulated 64-bit counters start at 1e14 so history fits below them.
|
||||
|
||||
**oidc-mock** runs `internal/auth/oidctest` as a standalone OpenID Connect issuer for the M6 gate and for screenshots:
|
||||
|
||||
nmon-devtools oidc-mock --issuer-file F --port-file P [--client-id nmon --client-secret s3cret]
|
||||
curl 'http://[::1]:PORT/user?sub=viewer-1&name=Vera&groups=staff' # who the next login is
|
||||
|
||||
The issuer approves every login at once as the current user (the control endpoint changes it); `--issuer-file` receives the issuer URL for `auth.oidc.issuer`. **bcrypt PASSWORD** prints a bcrypt hash, the content of a break-glass file. Every devtools command that calls the API sends `Authorization: Bearer $NMON_TOKEN` when that variable is set (an nmon with authentication on needs an admin token).
|
||||
|
||||
## Gates
|
||||
|
||||
`make gate-m3` runs everything above end to end and writes live screenshots to `docs/screenshots/M3/`.
|
||||
It needs Playwright's chromium: `cd web && npx playwright install chromium`.
|
||||
|
||||
`make gate-m6` starts nmon with a mock IdP and a break-glass file (authentication ON) and checks roles, tokens, break-glass lock-out, backup/restore, key rotation, settings reload, CSV import and the UI; its live screenshots go to `docs/screenshots/M6/`.
|
||||
|
||||
@@ -44,4 +44,13 @@ Alerting: all *Must* and *Must not* lines. Adversarial: redundant parents failin
|
||||
|
||||
## Gate log
|
||||
|
||||
_(filled in by the supervisor when the gate passes)_
|
||||
**Passed 2026-10-23**, run by the supervisor (Opus).
|
||||
|
||||
| Check | Result |
|
||||
|---|---|
|
||||
| `make lint` (Go + web) | 0 findings |
|
||||
| `go test -race ./...`, `-tags 'integration vmembed'`, `test-pg.sh` | all ok |
|
||||
| gates m1, m2 (LibreNMS recordings), m3, m4, m5 in one sequential run | all PASS |
|
||||
| `make gate-m4` | PASS: redundant topology (2 cores, 2 dual-homed distribution switches, 20 access switches) with `--icmp off`; (a) one core down → exactly one notice, nothing suppressed; (b) **both cores and all 22 children stopped at the same instant → exactly ONE notification** naming both cores and 22 suppressed children, zero per-child notices; (c) restart re-sends nothing; (d) core-1 back → one resolution, dist switches announced with their suppressed children; deliveries logged with redacted payloads; incident IDs stable |
|
||||
|
||||
**Decisions in M4:** a dependency hold delays a child's notice while its parents are unsettled (proven over 100 randomised timing seeds), so simultaneous outages group correctly. Unsupported ICMP falls back to SNMP alone and shows an `icmp_unsupported` device warning. Single-signal warnings are held 5 min. `--icmp auto|off`. `alerts?state=open` (open alerts, acked or not) is used on device pages. That fix came from the gate's screenshot step, which caught an acked outage vanishing from its device page.
|
||||
|
||||
@@ -45,4 +45,13 @@ Storage and rollups: all lines except the loadtest disk budget (M6). Ingest: all
|
||||
|
||||
## Gate log
|
||||
|
||||
_(filled in by the supervisor when the gate passes)_
|
||||
**Passed 2026-10-23**, run by the supervisor (Opus).
|
||||
|
||||
| Check | Result |
|
||||
|---|---|
|
||||
| `make lint` (Go + web) | 0 findings |
|
||||
| `go test -race ./...`, `-tags 'integration vmembed'`, `test-pg.sh` | all ok |
|
||||
| gates m1, m2 (LibreNMS recordings), m3, m4, m5 in one sequential run | all PASS |
|
||||
| `make gate-m5` | PASS: mock Proxmox synced (node pve1 + guests with hosted_on); `proxmox.influx` replayed via a push token → 93 series under pve1; **2 y graph hits the 1 h tier** (tiers `1h`+`raw`, `stitched_at`, long points exactly the back-filled values); p95 with `source`; group graph is the oriented sum; HTTP check up; ingest negatives (401s, reserved prefix dropped, series-cap alert deduped) |
|
||||
|
||||
**Decisions in M5:** see DEVIATIONS rows 18–30 (push token format, rollup `pending` status and `v` encoding, gauge rollup names, month-partitioned 5-min retention, acked synchronous long writes, unreliable interfaces never rolled up, group `coverage` label, check `check` label, Proxmox node rule, `push="1"` label, system alarms, no inline Proxmox secret). One `internal/longq` package now owns the "highest version wins" query; this fixed M3's long-tier graph read.
|
||||
|
||||
@@ -25,11 +25,6 @@
|
||||
- `nmon loadtest`: in-process simagents (2,000 × 100 ports on `::1` with distinct ports), nmon serve in-process, traps plus push plus rollup running, metrics collected from `nmon_` series and `/proc/self/status`.
|
||||
- Platforms: `make build-arm64` cross-compiles. Non-Linux builds compile without `vmembed` and require `--vm-*-url`.
|
||||
|
||||
### Decisions recorded during M6 (see `docs/DEVIATIONS.md` rows 31-37)
|
||||
- **No login UI** (owner decision): page loads without a session redirect (302) to `/auth/login?return=...` (OIDC); API calls get 401; break-glass is `GET /auth/breakglass` with HTTP Basic; the SPA has a user menu (name, role, sign out) and role-aware controls but no login page. Dev-open mode is unchanged.
|
||||
- Hot reload covers receivers, the outbound allow-list, trusted proxies, backup schedule, disk thresholds and the purge period; the rest is `restart_required`.
|
||||
- Interfaces gained an operator-owned `description` (migration 8) that the CSV port import writes and discovery never overwrites.
|
||||
|
||||
## Work packages
|
||||
|
||||
| WP | Files | Depends on |
|
||||
|
||||
@@ -15,9 +15,9 @@ its **Gate log** is filled in when the gate passes. A milestone is done only whe
|
||||
| M1 | Skeleton and fixtures | `nmon serve` polls the counters fixture and `/api/v1/query` returns rising ifHCInOctets | [M1](M1-skeleton-and-fixtures.md) | **passed**, tagged `m1` |
|
||||
| M2 | Discovery and profiles | every fixture's expect.yaml passes, including v1 and v3, and the PDU budget holds | [M2](M2-discovery-and-profiles.md) | **passed**, tagged `m2` |
|
||||
| M3 | Graphs and UI | a port's in/out graph renders at 1 h, 24 h and 30 d from the counters fixture | [M3](M3-graphs-and-ui.md) | **passed**, tagged `m3` |
|
||||
| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | in progress (WP1 reach and deps, WP2 rules, WP3 notify done) |
|
||||
| M5 | Push ingest, rollup, checks | a replayed Proxmox sample shows under its node and a 2 y query hits the 1 h tier | [M5](M5-ingest-rollup-checks.md) | planned |
|
||||
| M6 | Auth and ops | a viewer gets 403 on PUT (2,000-device loadtest skipped, approved deviation) | [M6](M6-auth-and-ops.md) | planned |
|
||||
| M4 | Alerting | stopping a parent fixture sends one notification naming it and its children | [M4](M4-alerting.md) | **passed**, tagged `m4` |
|
||||
| M5 | Push ingest, rollup, checks | a replayed Proxmox sample shows under its node and a 2 y query hits the 1 h tier | [M5](M5-ingest-rollup-checks.md) | **passed**, tagged `m5` |
|
||||
| M6 | Auth and ops | a viewer gets 403 on PUT (2,000-device loadtest skipped, approved deviation) | [M6](M6-auth-and-ops.md) | in progress |
|
||||
| M7 | gNMI streaming | after v1; not built in this run | — | out of scope |
|
||||
|
||||
## How the work is run
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
# Operating nmon
|
||||
|
||||
How to run, secure, back up and maintain a production nmon. Design background: `docs/spec/spec.md` ("Auth", "Operations").
|
||||
|
||||
## Starting
|
||||
|
||||
nmon serve --data-dir /var/lib/nmon --secret-key-file /etc/nmon/secret.key \
|
||||
--tls-cert /etc/nmon/tls.crt --tls-key /etc/nmon/tls.key
|
||||
|
||||
- **HTTPS is the default.** Pass a certificate pair (`--tls-cert`, `--tls-key`; `SIGHUP` reloads them without dropping connections) or Let's Encrypt (`--acme-host nmon.example.net` repeatable, `--acme-email you@example.net`; nmon answers the http-01 challenge on `--acme-listen`, default `:80`, certificates are cached in `<data-dir>/acme`). Without either, `serve` refuses to start and says so. `--insecure-http` is for development: plain HTTP, cookies without `Secure`.
|
||||
- **A sign-in method is required** over TLS (`auth.oidc` and/or `auth.break_glass_file` in `config.yaml`). With `--insecure-http` and neither configured nmon runs in *dev-open mode* (everyone is admin `dev`; a banner says so); never expose that port.
|
||||
- Behind a reverse proxy, list it in `auth.trusted_proxies`; `X-Forwarded-*` is ignored from anyone else.
|
||||
|
||||
## Sign-in
|
||||
|
||||
`config.yaml`:
|
||||
|
||||
auth:
|
||||
oidc:
|
||||
issuer: https://idp.example.net/realms/net
|
||||
client_id: nmon
|
||||
client_secret_name: oidc-client # the secret itself is never in this file
|
||||
redirect_url: https://nmon.example.net/auth/callback
|
||||
groups_claim: groups
|
||||
admin_group: nmon-admins # members are admins; every other user is a viewer
|
||||
break_glass_file: /etc/nmon/breakglass # bcrypt hash, "hash" or "user:hash", created by you
|
||||
|
||||
Store the client secret before the first start: `printf '%s\n' "$SECRET" | nmon secret set oidc-client --data-dir D --secret-key-file K` (it can also be set later with `PUT /api/v1/secrets/oidc-client`; OIDC stays off until the secret exists). The break-glass file holds one bcrypt hash (`$2a$`/`$2b$`), made with any bcrypt tool (the source tree's `nmon-devtools bcrypt PASSWORD` works); keep it readable by the nmon user only.
|
||||
|
||||
- There is no login page. Opening nmon without a session redirects to the IdP and back; API clients use a token. If the IdP is down, open `/auth/breakglass`: the browser asks for the local admin user and password (HTTP Basic), and a successful sign-in lands on the page you asked for.
|
||||
- Break-glass: five wrong passwords lock the source address for 15 minutes; every use and every lock is audited and raises an alert. `nmon unlock [IP|all] --data-dir D` clears locks, also while nmon runs.
|
||||
- API tokens: an admin mints viewer, admin or push tokens (`POST /api/v1/tokens`, or Settings, API tokens). Use `Authorization: Bearer nma_...`. A push token works for ingest only. Revoking takes effect on the next request.
|
||||
- `GET /api/v1/audit` (admin) lists who did what.
|
||||
|
||||
## Backup and restore
|
||||
|
||||
A running nmon owns the stores, so copying the data folder is **not** a backup.
|
||||
|
||||
nmon backup /backups/nmon.tar --api https://nmon.example.net --token nma_... # key NOT included
|
||||
nmon backup /backups/nmon.tar --include-key --api ... --token ... # key included
|
||||
|
||||
Every run prints whether the key file is in the archive. **Lose the key file and a restore has an inventory with no SNMP credentials.** The nightly schedule is `backup: {at: "02:30", dir: /var/backups/nmon, keep: 7}` (time in the configured timezone, default dir `<data-dir>/backups`); a failed run raises the `backup_failed` alert. A backup is refused when the disk is below its 5% reserve.
|
||||
|
||||
Restore with nmon stopped, into a clean directory:
|
||||
|
||||
nmon restore /backups/nmon.tar --data-dir /var/lib/nmon-restored --secret-key-file /etc/nmon/secret.key
|
||||
|
||||
It verifies inventory row counts and a sample of series before it says done. Then start nmon on that directory. Restore needs the embedded stores (the normal release build).
|
||||
|
||||
## Keys
|
||||
|
||||
`nmon key rotate --data-dir D --secret-key-file K` (nmon stopped) adds a new key version, re-encrypts every SNMP credential and secret under it, and keeps the old key in the file so older backups still restore. Remove old keys only after a release.
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit in Settings (admin) or `config.yaml` + `SIGHUP`. An invalid document is rejected and the running configuration stays. Alert receivers, the outbound allow-list, trusted proxies, the backup schedule, disk thresholds and the purge period apply live; everything else is listed as `restart_required` until the next restart. `disk: {alert_used: 0.8, admit_used: 0.9, reserve_free: 0.05}`: alert at 80% used, no new series at 90%, ingest throttled / rollup paused / backups refused below 5% free.
|
||||
|
||||
## Metric stores
|
||||
|
||||
The supervisor restarts a crashed store; after three restarts in ten minutes it raises `store_failed` and stops. `nmon store restart [hot|long] --api URL --token T` (or `SIGHUP`) resumes it.
|
||||
|
||||
## Retired devices
|
||||
|
||||
`POST /api/v1/devices/{slug}/purge` (admin) deletes a retired device's series from both stores once it has been retired for `retired_purge_after` (default 1y).
|
||||
|
||||
## CSV import
|
||||
|
||||
Devices (`name,address,site,role,credential,parents,...`) and port descriptions (`device,ifname,alias`) from Settings or `POST /api/v1/devices/import`. Port descriptions are stored in the operator-owned `description` field, which discovery never overwrites. A dependency cycle is rejected with its path; nothing is written unless the whole import is valid.
|
||||
|
Before Width: | Height: | Size: 126 KiB After Width: | Height: | Size: 126 KiB |
|
Before Width: | Height: | Size: 97 KiB After Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 594 KiB After Width: | Height: | Size: 593 KiB |
|
Before Width: | Height: | Size: 161 KiB After Width: | Height: | Size: 162 KiB |
|
Before Width: | Height: | Size: 122 KiB After Width: | Height: | Size: 122 KiB |
|
Before Width: | Height: | Size: 160 KiB After Width: | Height: | Size: 158 KiB |
|
Before Width: | Height: | Size: 123 KiB After Width: | Height: | Size: 122 KiB |
|
Before Width: | Height: | Size: 88 KiB After Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 88 KiB After Width: | Height: | Size: 87 KiB |
|
Before Width: | Height: | Size: 73 KiB After Width: | Height: | Size: 72 KiB |
|
Before Width: | Height: | Size: 111 KiB After Width: | Height: | Size: 111 KiB |
|
Before Width: | Height: | Size: 112 KiB After Width: | Height: | Size: 112 KiB |
|
Before Width: | Height: | Size: 158 KiB After Width: | Height: | Size: 158 KiB |
|
Before Width: | Height: | Size: 42 KiB After Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 110 KiB After Width: | Height: | Size: 110 KiB |
|
Before Width: | Height: | Size: 44 KiB After Width: | Height: | Size: 44 KiB |
|
Before Width: | Height: | Size: 51 KiB After Width: | Height: | Size: 51 KiB |
|
Before Width: | Height: | Size: 53 KiB After Width: | Height: | Size: 52 KiB |
|
Before Width: | Height: | Size: 85 KiB After Width: | Height: | Size: 85 KiB |
|
Before Width: | Height: | Size: 189 KiB After Width: | Height: | Size: 186 KiB |
|
Before Width: | Height: | Size: 49 KiB After Width: | Height: | Size: 48 KiB |
|
Before Width: | Height: | Size: 213 KiB After Width: | Height: | Size: 212 KiB |
|
Before Width: | Height: | Size: 58 KiB After Width: | Height: | Size: 57 KiB |
|
Before Width: | Height: | Size: 80 KiB After Width: | Height: | Size: 80 KiB |
|
Before Width: | Height: | Size: 88 KiB After Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 175 KiB After Width: | Height: | Size: 175 KiB |
|
Before Width: | Height: | Size: 63 KiB After Width: | Height: | Size: 63 KiB |
|
Before Width: | Height: | Size: 84 KiB After Width: | Height: | Size: 84 KiB |
|
Before Width: | Height: | Size: 65 KiB After Width: | Height: | Size: 65 KiB |
|
Before Width: | Height: | Size: 48 KiB After Width: | Height: | Size: 48 KiB |
|
Before Width: | Height: | Size: 53 KiB After Width: | Height: | Size: 53 KiB |
|
Before Width: | Height: | Size: 83 KiB After Width: | Height: | Size: 83 KiB |
|
Before Width: | Height: | Size: 96 KiB After Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 34 KiB After Width: | Height: | Size: 34 KiB |
|
Before Width: | Height: | Size: 81 KiB After Width: | Height: | Size: 81 KiB |
|
Before Width: | Height: | Size: 42 KiB After Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 174 KiB |
|
Before Width: | Height: | Size: 80 KiB |
|
Before Width: | Height: | Size: 67 KiB |
|
Before Width: | Height: | Size: 153 KiB |
|
Before Width: | Height: | Size: 86 KiB |
|
Before Width: | Height: | Size: 79 KiB |
|
Before Width: | Height: | Size: 53 KiB |
|
Before Width: | Height: | Size: 77 KiB |
@@ -14,18 +14,9 @@ const (
|
||||
RuleP95Late = "p95_late"
|
||||
// RuleRollupStalled is the rollup job failing for longer than its tolerance.
|
||||
RuleRollupStalled = "rollup_stalled"
|
||||
// RuleBreakGlassPrefix starts the break-glass alarms: breakglass_login, breakglass_failure, breakglass_locked.
|
||||
RuleBreakGlassPrefix = "breakglass_"
|
||||
// RuleBackupFailed is the nightly backup failing; it closes when the next backup succeeds.
|
||||
RuleBackupFailed = "backup_failed"
|
||||
)
|
||||
|
||||
// SystemRule reports whether name is a subsystem alarm rule.
|
||||
func SystemRule(name string) bool {
|
||||
return name == RuleP95Late || name == RuleBackupFailed || name == RuleRollupStalled || strings.HasPrefix(name, RuleIngestPrefix) ||
|
||||
diskGuardRules[name] || strings.HasPrefix(name, RuleBreakGlassPrefix)
|
||||
return name == RuleP95Late || name == RuleRollupStalled || strings.HasPrefix(name, RuleIngestPrefix)
|
||||
}
|
||||
|
||||
// diskGuardRules are the disk guard's alarm kinds (internal/diskguard). They are listed rather than matched by a
|
||||
// "disk_" prefix because the shipped metric rule "disk_full" shares it.
|
||||
var diskGuardRules = map[string]bool{"disk_high": true, "disk_series_halt": true, "disk_reserve": true, "disk_statfs": true}
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
)
|
||||
|
||||
// anonymousActor names the caller when no auth middleware ran (unit tests that call the API directly).
|
||||
const anonymousActor = "anonymous"
|
||||
|
||||
// actorOf names the authenticated caller for the audit log, acknowledgements and maintenance windows:
|
||||
// the OIDC subject, "token:<name>" for an API token, "breakglass", or "dev" in development-open mode.
|
||||
func actorOf(ctx context.Context) string {
|
||||
if p, ok := auth.FromContext(ctx); ok {
|
||||
return p.Actor()
|
||||
}
|
||||
return anonymousActor
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
"github.com/danielgtaylor/huma/v2/adapters/humago"
|
||||
|
||||
"github.com/v10networks/nmon/internal/vmsup"
|
||||
)
|
||||
|
||||
// ErrBackupRefused means a backup was refused because the disk is below the reserve.
|
||||
var ErrBackupRefused = errors.New("backup refused")
|
||||
|
||||
// BackupFile is a finished backup waiting to be streamed to the caller.
|
||||
type BackupFile struct {
|
||||
Path string
|
||||
KeyIncluded bool
|
||||
// Done deletes the file once it has been sent.
|
||||
Done func()
|
||||
}
|
||||
|
||||
// BackupRunner runs one online backup into a temporary file.
|
||||
type BackupRunner interface {
|
||||
Create(ctx context.Context, includeKey bool) (BackupFile, error)
|
||||
}
|
||||
|
||||
// StoreRestarter resumes a metric store that gave up after repeated crashes. *vmsup.Supervisor implements it.
|
||||
type StoreRestarter interface{ Restart(name string) error }
|
||||
|
||||
// backupOut streams the archive. KeyIncluded repeats what the archive's manifest says, so a client can print it.
|
||||
type backupOut struct {
|
||||
ContentType string `header:"Content-Type"`
|
||||
Disposition string `header:"Content-Disposition"`
|
||||
KeyIncluded string `header:"X-Nmon-Key-Included"`
|
||||
Body func(ctx huma.Context)
|
||||
}
|
||||
|
||||
func (s *server) registerAdmin(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "create-backup", Method: http.MethodPost, Path: "/api/v1/admin/backup", Tags: []string{"admin"},
|
||||
Summary: "Take a backup and download it",
|
||||
Description: "Admin only. Runs nmon's online backup (inventory snapshot, both metric stores' snapshots, config and profiles) " +
|
||||
"and streams the tar. The X-Nmon-Key-Included header says whether the encryption key is inside. `nmon backup` is the client.",
|
||||
Responses: map[string]*huma.Response{"200": {
|
||||
Description: "The backup archive.",
|
||||
Content: map[string]*huma.MediaType{"application/x-tar": {}},
|
||||
}},
|
||||
Middlewares: huma.Middlewares{noWriteDeadline},
|
||||
Errors: []int{http.StatusInsufficientStorage, http.StatusServiceUnavailable},
|
||||
}, s.createBackup)
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "restart-store", Method: http.MethodPost, Path: "/api/v1/admin/stores/{name}/restart", Tags: []string{"admin"},
|
||||
Summary: "Resume a failed metric store", DefaultStatus: http.StatusNoContent,
|
||||
Description: "Admin only. After store_failed the supervisor stopped restarting the store; this starts it again. " +
|
||||
"A store that has not failed is a 409.",
|
||||
Errors: []int{http.StatusNotFound, http.StatusConflict, http.StatusServiceUnavailable},
|
||||
}, s.restartStore)
|
||||
}
|
||||
|
||||
// noWriteDeadline lifts the server's write timeout for one request: a backup can take longer than it.
|
||||
func noWriteDeadline(ctx huma.Context, next func(huma.Context)) {
|
||||
if _, w := humago.Unwrap(ctx); w != nil {
|
||||
_ = http.NewResponseController(w).SetWriteDeadline(time.Time{}) // unsupported writers keep the default
|
||||
}
|
||||
next(ctx)
|
||||
}
|
||||
|
||||
func (s *server) createBackup(ctx context.Context, in *struct {
|
||||
IncludeKey bool `query:"include_key" doc:"Put the encryption key in the archive. Default false."`
|
||||
}) (*backupOut, error) {
|
||||
if s.Backups == nil {
|
||||
return nil, errNotConfigured("backup")
|
||||
}
|
||||
bf, err := s.Backups.Create(ctx, in.IncludeKey)
|
||||
if errors.Is(err, ErrBackupRefused) {
|
||||
return nil, huma.NewError(http.StatusInsufficientStorage, trimSentinel(err, ErrBackupRefused))
|
||||
}
|
||||
if err != nil {
|
||||
s.Logger.Error("backup", "err", err)
|
||||
return nil, huma.Error500InternalServerError("backup failed; see the log")
|
||||
}
|
||||
detail := map[string]bool{"key_included": bf.KeyIncluded}
|
||||
if err := s.Store.AppendAudit(ctx, actorOf(ctx), "backup.create", "backup", nil, detail); err != nil {
|
||||
s.Logger.Warn("audit backup.create", "err", err)
|
||||
}
|
||||
key := "false"
|
||||
if bf.KeyIncluded {
|
||||
key = "true"
|
||||
}
|
||||
return &backupOut{
|
||||
ContentType: "application/x-tar", Disposition: `attachment; filename="nmon-backup.tar"`, KeyIncluded: key,
|
||||
Body: func(hc huma.Context) { s.streamBackup(hc, bf) },
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *server) streamBackup(hc huma.Context, bf BackupFile) {
|
||||
defer bf.Done()
|
||||
f, err := os.Open(bf.Path)
|
||||
if err != nil {
|
||||
s.Logger.Error("open backup for download", "err", err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
if _, err := io.Copy(hc.BodyWriter(), f); err != nil {
|
||||
s.Logger.Warn("stream backup", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *server) restartStore(ctx context.Context, in *struct {
|
||||
Name string `path:"name" enum:"hot,long" doc:"Which metric store."`
|
||||
}) (*struct{}, error) {
|
||||
if s.Stores == nil {
|
||||
return nil, errNotConfigured("metric store supervisor")
|
||||
}
|
||||
err := s.Stores.Restart(in.Name)
|
||||
switch {
|
||||
case errors.Is(err, vmsup.ErrUnknownStore):
|
||||
return nil, huma.Error404NotFound("no such store")
|
||||
case errors.Is(err, vmsup.ErrNotFailed), errors.Is(err, vmsup.ErrExternalStore):
|
||||
return nil, huma.Error409Conflict(err.Error())
|
||||
case err != nil:
|
||||
s.Logger.Error("restart store", "err", err)
|
||||
return nil, huma.Error500InternalServerError("restart failed; see the log")
|
||||
}
|
||||
if err := s.Store.AppendAudit(ctx, actorOf(ctx), "store.restart", "store:"+in.Name, nil, nil); err != nil {
|
||||
s.Logger.Warn("audit store.restart", "err", err)
|
||||
}
|
||||
return &struct{}{}, nil
|
||||
}
|
||||
@@ -176,10 +176,10 @@ func (s *server) ackAlert(ctx context.Context, in *struct {
|
||||
return nil, huma.Error422UnprocessableEntity("until must be in the future")
|
||||
}
|
||||
err = s.Store.WithTx(ctx, func(tx store.Store) error {
|
||||
if err := tx.AckAlert(ctx, id, actorOf(ctx), in.Body.Reason, in.Body.Until.UTC()); err != nil {
|
||||
if err := tx.AckAlert(ctx, id, "anonymous", in.Body.Reason, in.Body.Until.UTC()); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "alert.ack", "alert:"+in.ID, nil,
|
||||
return tx.AppendAudit(ctx, "anonymous", "alert.ack", "alert:"+in.ID, nil,
|
||||
map[string]any{"reason": in.Body.Reason, "until": in.Body.Until.UTC()})
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -103,22 +103,6 @@ type Deps struct {
|
||||
Zones p95.Zones
|
||||
Checks CheckResults
|
||||
Proxmox ProxmoxStatus
|
||||
|
||||
// M6 collaborators; a route whose collaborator is nil answers 503 "not configured".
|
||||
// APITokens mints and revokes viewer and admin API tokens (scope push stays with Tokens).
|
||||
APITokens APITokenAdmin
|
||||
// Settings reads and replaces config.yaml and reloads it.
|
||||
Settings SettingsAdmin
|
||||
// Backups runs an online backup for POST /admin/backup.
|
||||
Backups BackupRunner
|
||||
// Stores restarts a failed metric store.
|
||||
Stores StoreRestarter
|
||||
// Series deletes whole series from the metric stores (device purge).
|
||||
Series SeriesDeleter
|
||||
// PurgeAfter is how long a device must have been retired before it can be purged. Default one year.
|
||||
PurgeAfter func() time.Duration
|
||||
// OnImported is told the ids of devices a CSV import created, so they can be discovered at once.
|
||||
OnImported func(ids []int64)
|
||||
}
|
||||
|
||||
type server struct {
|
||||
@@ -203,12 +187,6 @@ func (s *server) register(api huma.API) {
|
||||
s.registerP95(api)
|
||||
s.registerRollup(api)
|
||||
s.registerProxmox(api)
|
||||
s.registerMe(api)
|
||||
s.registerSettings(api)
|
||||
s.registerImport(api)
|
||||
s.registerPurge(api)
|
||||
s.registerAdmin(api)
|
||||
s.registerAudit(api)
|
||||
}
|
||||
|
||||
// newHuma builds the huma API on mux. /docs is served by registerDocs from embedded assets, not huma's CDN page.
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// AuditView is one audit-log entry. What changed (before and after) stays in the database: it can hold request
|
||||
// bodies, and this list is for answering who did what and when.
|
||||
type AuditView struct {
|
||||
At time.Time `json:"at"`
|
||||
Actor string `json:"actor" doc:"Who: an IDP subject, token:<name>, breakglass, console, dev or system."`
|
||||
Action string `json:"action" example:"device.edit"`
|
||||
Object string `json:"object"`
|
||||
}
|
||||
|
||||
// AuditList is a page of AuditView, newest first.
|
||||
type AuditList struct {
|
||||
Items []AuditView `json:"items"`
|
||||
Page int64 `json:"page"`
|
||||
PerPage int64 `json:"per_page"`
|
||||
Total int64 `json:"total"`
|
||||
}
|
||||
|
||||
func (s *server) registerAudit(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "list-audit", Method: http.MethodGet, Path: "/api/v1/audit", Tags: []string{"admin"},
|
||||
Summary: "Audit log",
|
||||
Description: "Admin only. Who changed what, newest first: logins, token and secret changes, imports, settings, backups.",
|
||||
}, s.listAudit)
|
||||
}
|
||||
|
||||
func (s *server) listAudit(ctx context.Context, in *struct{ PageParams }) (*struct{ Body AuditList }, error) {
|
||||
rows, total, err := s.Store.ListAudit(ctx, store.Page{Offset: in.offset(), Limit: in.limit()})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
out := AuditList{Items: make([]AuditView, 0, len(rows)), Page: in.pageNo(), PerPage: in.limit(), Total: total}
|
||||
for _, r := range rows {
|
||||
out.Items = append(out.Items, AuditView{At: r.At.UTC(), Actor: r.Actor, Action: r.Action, Object: r.Object})
|
||||
}
|
||||
return &struct{ Body AuditList }{out}, nil
|
||||
}
|
||||
@@ -199,7 +199,7 @@ func (s *server) createCheck(ctx context.Context, in *struct{ Body CheckBody })
|
||||
return err
|
||||
}
|
||||
row = c
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "check.create", "check:"+strconv.FormatInt(c.ID, 10), nil, in.Body)
|
||||
return tx.AppendAudit(ctx, "anonymous", "check.create", "check:"+strconv.FormatInt(c.ID, 10), nil, in.Body)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.txErr(err)
|
||||
@@ -230,7 +230,7 @@ func (s *server) updateCheck(ctx context.Context, in *struct {
|
||||
return err
|
||||
}
|
||||
row = c
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "check.update", "check:"+in.ID, nil, in.Body)
|
||||
return tx.AppendAudit(ctx, "anonymous", "check.update", "check:"+in.ID, nil, in.Body)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.txErr(err)
|
||||
@@ -250,7 +250,7 @@ func (s *server) deleteCheck(ctx context.Context, in *struct {
|
||||
if err := tx.DeleteCheck(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "check.delete", "check:"+in.ID, nil, nil)
|
||||
return tx.AppendAudit(ctx, "anonymous", "check.delete", "check:"+in.ID, nil, nil)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.txErr(err)
|
||||
|
||||
@@ -125,7 +125,7 @@ func (s *server) applyEdit(ctx context.Context, dev *store.Device, p *editPlan)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "device.edit", dev.Slug, nil, p.change)
|
||||
return tx.AppendAudit(ctx, "anonymous", "device.edit", dev.Slug, nil, p.change)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,6 @@ type InterfaceView struct {
|
||||
IfName string `json:"ifname"`
|
||||
IfDescr string `json:"ifdescr"`
|
||||
IfAlias string `json:"ifalias"`
|
||||
Description string `json:"description" doc:"Operator-owned port description; discovery never overwrites it."`
|
||||
IfPhysAddr string `json:"ifphysaddr"`
|
||||
IfSpeed int64 `json:"ifspeed" doc:"bits per second"`
|
||||
IfHighSpeed int64 `json:"ifhighspeed" doc:"megabits per second"`
|
||||
@@ -199,7 +198,7 @@ func interfaceView(i *store.Interface, r *ifRates) InterfaceView {
|
||||
in, out := r.bps(i)
|
||||
inE, outE := r.errs(i)
|
||||
return InterfaceView{
|
||||
ID: i.ID, IfIndex: i.IfIndex, IfName: i.IfName, IfDescr: i.IfDescr, IfAlias: i.IfAlias, Description: i.Description,
|
||||
ID: i.ID, IfIndex: i.IfIndex, IfName: i.IfName, IfDescr: i.IfDescr, IfAlias: i.IfAlias,
|
||||
IfPhysAddr: i.IfPhysAddr, IfSpeed: i.IfSpeed, IfHighSpeed: i.IfHighSpeed, IfType: i.IfType,
|
||||
AdminStatus: i.AdminStatus, OperStatus: i.OperStatus, HasHCCounters: i.HasHCCounters, Excluded: i.Excluded,
|
||||
FirstSeen: i.FirstSeen.UTC(), LastDiscovered: i.LastDiscovered.UTC(), Unreliable: i.Unreliable,
|
||||
|
||||
@@ -212,7 +212,7 @@ func (s *server) persistDevice(
|
||||
if restore != nil {
|
||||
action = "device.reattach"
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), action, d.Slug, nil,
|
||||
return tx.AppendAudit(ctx, "anonymous", action, d.Slug, nil,
|
||||
map[string]any{"name": d.Name, "kind": d.Kind, "addresses": req.strs})
|
||||
})
|
||||
return slug, err
|
||||
|
||||
@@ -150,7 +150,7 @@ func (s *server) createGroup(ctx context.Context, in *struct{ Body GroupBody })
|
||||
if err := tx.CreateGroup(ctx, &g); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "group.create", "group:"+strconv.FormatInt(g.ID, 10), nil, in.Body)
|
||||
return tx.AppendAudit(ctx, "anonymous", "group.create", "group:"+strconv.FormatInt(g.ID, 10), nil, in.Body)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
@@ -230,7 +230,7 @@ func (s *server) updateGroup(ctx context.Context, in *struct {
|
||||
if err := tx.UpdateGroup(ctx, g); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "group.update", "group:"+in.ID, nil, in.Body)
|
||||
return tx.AppendAudit(ctx, "anonymous", "group.update", "group:"+in.ID, nil, in.Body)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
@@ -253,7 +253,7 @@ func (s *server) deleteGroup(ctx context.Context, in *struct {
|
||||
if err := tx.DeleteGroup(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "group.delete", "group:"+in.ID, nil, nil)
|
||||
return tx.AppendAudit(ctx, "anonymous", "group.delete", "group:"+in.ID, nil, nil)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
@@ -282,7 +282,7 @@ func (s *server) addGroupMember(ctx context.Context, in *struct {
|
||||
if err := tx.AddGroupMember(ctx, m); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "group.member.add", "group:"+in.ID, nil, in.Body)
|
||||
return tx.AppendAudit(ctx, "anonymous", "group.member.add", "group:"+in.ID, nil, in.Body)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
@@ -307,7 +307,7 @@ func (s *server) endGroupMember(ctx context.Context, in *struct {
|
||||
if err := tx.EndGroupMember(ctx, id, in.InterfaceID, at); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "group.member.end", "group:"+in.ID, nil, map[string]string{"interface_id": in.InterfaceID})
|
||||
return tx.AppendAudit(ctx, "anonymous", "group.member.end", "group:"+in.ID, nil, map[string]string{"interface_id": in.InterfaceID})
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/deps"
|
||||
"github.com/v10networks/nmon/internal/importcsv"
|
||||
)
|
||||
|
||||
// importMaxBytes bounds the multipart body of one import.
|
||||
const importMaxBytes = 8 << 20
|
||||
|
||||
// ImportForm is the multipart form of POST /devices/import; send either file or both.
|
||||
type ImportForm struct {
|
||||
Devices huma.FormFile `form:"devices" contentType:"text/csv,text/plain,application/csv,application/vnd.ms-excel,application/octet-stream" required:"false" doc:"devices.csv: name, address, site, role, credential, parents."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
|
||||
Ports huma.FormFile `form:"ports" contentType:"text/csv,text/plain,application/csv,application/vnd.ms-excel,application/octet-stream" required:"false" doc:"ports.csv: device, ifname, alias (stored as the port description)."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
|
||||
}
|
||||
|
||||
// ImportResult counts what an import wrote.
|
||||
type ImportResult struct {
|
||||
DevicesCreated int `json:"devices_created"`
|
||||
SNMPProfilesCreated int `json:"snmp_profiles_created" doc:"SNMP credential records created for devices that gave a community inline."`
|
||||
PortsUpdated int `json:"ports_updated" doc:"Interfaces whose operator-owned description was set."`
|
||||
}
|
||||
|
||||
func (s *server) registerImport(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "import-devices", Method: http.MethodPost, Path: "/api/v1/devices/import", Tags: []string{"devices"},
|
||||
Summary: "Import devices and port descriptions from CSV",
|
||||
Description: "Admin only. Multipart form with devices.csv and/or ports.csv. Every problem is reported with its row; " +
|
||||
"a dependency cycle is rejected with its path. Nothing is written unless the whole import is valid. " +
|
||||
"Port descriptions are operator-owned: discovery never overwrites them.",
|
||||
MaxBodyBytes: importMaxBytes,
|
||||
Errors: []int{http.StatusUnprocessableEntity},
|
||||
}, s.importDevices)
|
||||
}
|
||||
|
||||
func (s *server) importDevices(ctx context.Context, in *struct {
|
||||
RawBody huma.MultipartFormFiles[ImportForm]
|
||||
}) (*struct{ Body ImportResult }, error) {
|
||||
form := in.RawBody.Data()
|
||||
var inputs importcsv.Inputs
|
||||
if form.Devices.IsSet {
|
||||
inputs.Devices = form.Devices
|
||||
}
|
||||
if form.Ports.IsSet {
|
||||
inputs.Ports = form.Ports
|
||||
}
|
||||
if inputs.Devices == nil && inputs.Ports == nil {
|
||||
return nil, huma.Error422UnprocessableEntity("send devices.csv, ports.csv or both")
|
||||
}
|
||||
plan, err := importcsv.Build(ctx, s.Store, inputs)
|
||||
if err != nil {
|
||||
return nil, s.importErr(err)
|
||||
}
|
||||
res, err := importcsv.Apply(ctx, s.Store, plan)
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
if err := s.Store.AppendAudit(ctx, actorOf(ctx), "device.import", "import", nil, res); err != nil {
|
||||
s.Logger.Warn("audit device.import", "err", err)
|
||||
}
|
||||
s.afterImport(ctx, res)
|
||||
return &struct{ Body ImportResult }{ImportResult{
|
||||
DevicesCreated: res.DevicesCreated, SNMPProfilesCreated: res.CredentialsCreated, PortsUpdated: res.PortsUpdated,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// afterImport makes the poller and the dependency graph see the new devices and starts their discovery.
|
||||
func (s *server) afterImport(ctx context.Context, res importcsv.Result) {
|
||||
if s.Poller != nil {
|
||||
s.Poller.Refresh()
|
||||
}
|
||||
if err := s.Dependencies.Refresh(ctx); err != nil {
|
||||
s.Logger.Warn("refresh dependency graph after import", "err", err)
|
||||
}
|
||||
if s.OnImported != nil && len(res.DeviceIDs) > 0 {
|
||||
s.OnImported(res.DeviceIDs)
|
||||
}
|
||||
}
|
||||
|
||||
// importErr turns a plan failure into a 422 that names every bad row, or the cycle's path.
|
||||
func (s *server) importErr(err error) error {
|
||||
var pe *importcsv.PlanError
|
||||
if errors.As(err, &pe) {
|
||||
details := make([]error, 0, len(pe.Errors))
|
||||
for _, e := range pe.Errors {
|
||||
details = append(details, &huma.ErrorDetail{Message: e.Msg, Location: fmt.Sprintf("%s.csv row %d", e.File, e.Row)})
|
||||
}
|
||||
msg := fmt.Sprintf("%d row(s) are invalid; nothing was imported", len(pe.Errors))
|
||||
return huma.NewError(http.StatusUnprocessableEntity, msg, details...)
|
||||
}
|
||||
var ce *deps.CycleError
|
||||
if errors.As(err, &ce) {
|
||||
return huma.Error422UnprocessableEntity("nothing was imported: " + err.Error())
|
||||
}
|
||||
return s.storeErr(err)
|
||||
}
|
||||
@@ -15,7 +15,6 @@ type InterfaceHit struct {
|
||||
Device DeviceRef `json:"device"`
|
||||
IfName string `json:"ifname"`
|
||||
IfAlias string `json:"ifalias"`
|
||||
Description string `json:"description" doc:"Operator-owned port description (CSV import); discovery never overwrites it."`
|
||||
IfDescr string `json:"ifdescr"`
|
||||
OperStatus int64 `json:"oper_status"`
|
||||
AdminStatus int64 `json:"admin_status"`
|
||||
@@ -35,14 +34,13 @@ type InterfaceList struct {
|
||||
func (s *server) registerInterfaces(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "list-interfaces", Method: http.MethodGet, Path: "/api/v1/interfaces", Tags: []string{"interfaces"},
|
||||
Summary: "Search interfaces by name or alias",
|
||||
Description: "q matches ifname, ifalias and description, case-insensitively, across all devices. " +
|
||||
"Rates are null when unknown or unreliable.",
|
||||
Summary: "Search interfaces by name or alias",
|
||||
Description: "q matches ifname and ifalias, case-insensitively, across all devices. Rates are null when unknown or unreliable.",
|
||||
}, s.listInterfaces)
|
||||
}
|
||||
|
||||
func (s *server) listInterfaces(ctx context.Context, in *struct {
|
||||
Q string `query:"q" doc:"Substring of ifname, ifalias or description."`
|
||||
Q string `query:"q" doc:"Substring of ifname or ifalias."`
|
||||
PageParams
|
||||
}) (*struct{ Body InterfaceList }, error) {
|
||||
rows, total, err := s.Store.SearchInterfaceHits(ctx, in.Q, store.Page{Offset: in.offset(), Limit: in.limit()})
|
||||
@@ -62,7 +60,7 @@ func (s *server) listInterfaces(ctx context.Context, in *struct {
|
||||
r := &rows[i]
|
||||
inB, outB := rates[r.ID].bps(&r.Interface)
|
||||
out.Items = append(out.Items, InterfaceHit{
|
||||
ID: r.ID, Device: DeviceRef{Slug: r.DeviceSlug, Name: r.DeviceName}, IfName: r.IfName, IfAlias: r.IfAlias, Description: r.Description,
|
||||
ID: r.ID, Device: DeviceRef{Slug: r.DeviceSlug, Name: r.DeviceName}, IfName: r.IfName, IfAlias: r.IfAlias,
|
||||
IfDescr: r.IfDescr, OperStatus: r.OperStatus, AdminStatus: r.AdminStatus, SpeedBPS: speedBPS(&r.Interface),
|
||||
InBPS: inB, OutBPS: outB,
|
||||
})
|
||||
|
||||
@@ -1,354 +0,0 @@
|
||||
package api_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
"github.com/v10networks/nmon/internal/auth/authtest"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// call sends a request through the auth middleware with the given credentials.
|
||||
func (e *authedEnv) call(method, path, body string, c authtest.Creds) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(method, path, strings.NewReader(body))
|
||||
if body != "" {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if c.Token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.Token)
|
||||
}
|
||||
if c.Cookie != nil {
|
||||
req.AddCookie(c.Cookie)
|
||||
req.Header.Set(auth.CSRFHeader, c.CSRF)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
e.top.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestAPITokensOfEachKindMintListAndRevokeWithTheRightScope(t *testing.T) {
|
||||
e := newAuthedEnv(t, nil)
|
||||
adminSess, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
|
||||
viewerSess, _ := authtest.Mint(t, e.auth, auth.RoleViewer)
|
||||
dev := e.device(t, "core-1", "dc1", "core")
|
||||
|
||||
mint := func(body string) (id, secret string) {
|
||||
rec := e.call(http.MethodPost, "/api/v1/tokens", body, adminSess)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("mint %s: %d %s", body, rec.Code, rec.Body)
|
||||
}
|
||||
var v struct{ ID, Secret string }
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v.ID, v.Secret
|
||||
}
|
||||
vid, vsecret := mint(`{"name":"grafana","scope":"viewer"}`)
|
||||
_, asecret := mint(`{"name":"ci","scope":"admin"}`)
|
||||
_, psecret := mint(fmt.Sprintf(`{"name":"pve","scope":"push","devices":[%q]}`, dev.Slug))
|
||||
if !strings.HasPrefix(vsecret, "nma_") || !strings.HasPrefix(asecret, "nma_") || !strings.HasPrefix(psecret, "nmp_") {
|
||||
t.Fatalf("secret prefixes: %q %q %q", vsecret, asecret, psecret)
|
||||
}
|
||||
|
||||
viewer, admin, push := authtest.Creds{Token: vsecret}, authtest.Creds{Token: asecret}, authtest.Creds{Token: psecret}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
c authtest.Creds
|
||||
method string
|
||||
path string
|
||||
want int
|
||||
}{
|
||||
{"viewer token reads", viewer, http.MethodGet, "/api/v1/devices", http.StatusOK},
|
||||
{"viewer token cannot PUT", viewer, http.MethodPut, "/api/v1/devices/" + dev.Slug, http.StatusForbidden},
|
||||
{"viewer token cannot list tokens", viewer, http.MethodGet, "/api/v1/tokens", http.StatusForbidden},
|
||||
{"admin token PUTs", admin, http.MethodPut, "/api/v1/devices/" + dev.Slug, http.StatusOK},
|
||||
{"admin token lists tokens", admin, http.MethodGet, "/api/v1/tokens", http.StatusOK},
|
||||
{"push token cannot read", push, http.MethodGet, "/api/v1/devices", http.StatusForbidden},
|
||||
} {
|
||||
body := ""
|
||||
if tc.method == http.MethodPut {
|
||||
body = `{"name":"core-1"}`
|
||||
}
|
||||
if got := e.call(tc.method, tc.path, body, tc.c).Code; got != tc.want {
|
||||
t.Errorf("%s: %s %s = %d, want %d", tc.name, tc.method, tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
if code := e.call(http.MethodGet, "/api/v1/tokens", "", viewerSess).Code; code != http.StatusForbidden {
|
||||
t.Errorf("a viewer session listing tokens: %d, want 403", code)
|
||||
}
|
||||
|
||||
list := e.call(http.MethodGet, "/api/v1/tokens", "", admin)
|
||||
for _, secret := range []string{vsecret, asecret, psecret} {
|
||||
if strings.Contains(list.Body.String(), secret) || strings.Contains(list.Body.String(), "$") {
|
||||
t.Fatalf("the token list leaks a secret or hash: %s", list.Body)
|
||||
}
|
||||
}
|
||||
var items struct {
|
||||
Items []struct{ ID, Name, Scope string }
|
||||
}
|
||||
if err := json.Unmarshal(list.Body.Bytes(), &items); err != nil || len(items.Items) != 5 { // three minted here and the two Mint made for the sessions
|
||||
t.Fatalf("list: %v %s", err, list.Body)
|
||||
}
|
||||
|
||||
if code := e.call(http.MethodDelete, "/api/v1/tokens/"+vid, "", admin).Code; code != http.StatusNoContent {
|
||||
t.Fatalf("revoke the viewer token: %d", code)
|
||||
}
|
||||
if code := e.call(http.MethodGet, "/api/v1/devices", "", viewer).Code; code != http.StatusUnauthorized {
|
||||
t.Fatalf("a revoked token must be refused at once: %d", code)
|
||||
}
|
||||
if code := e.call(http.MethodGet, "/api/v1/devices", "", admin).Code; code != http.StatusOK {
|
||||
t.Fatalf("revoking one token must not affect another: %d", code)
|
||||
}
|
||||
if code := e.call(http.MethodPost, "/api/v1/tokens", `{"name":"x","scope":"admin","devices":["`+dev.Slug+`"]}`, adminSess).Code; code != http.StatusUnprocessableEntity {
|
||||
t.Fatalf("an admin token bound to devices: %d, want 422", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeReportsTheCaller(t *testing.T) {
|
||||
e := newAuthedEnv(t, nil)
|
||||
viewerSess, _ := authtest.Mint(t, e.auth, auth.RoleViewer)
|
||||
var me struct{ User, Role, Via, CSRF string }
|
||||
rec := e.call(http.MethodGet, "/api/v1/me", "", viewerSess)
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &me); err != nil || me.Role != "viewer" || me.Via != "session" || me.CSRF != viewerSess.CSRF {
|
||||
t.Fatalf("me = %+v (%v) %s", me, err, rec.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryAuditedActionNamesThePrincipalNotAnonymous(t *testing.T) {
|
||||
e := newAuthedEnv(t, nil)
|
||||
adminSess, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
|
||||
dev := e.device(t, "core-1", "dc1", "core")
|
||||
if rec := e.call(http.MethodPut, "/api/v1/devices/"+dev.Slug, `{"site":"dc2"}`, adminSess); rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
rows, _, err := e.st.ListAudit(context.Background(), store.Page{Limit: 50})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range rows {
|
||||
if r.Action == "device.edit" {
|
||||
found = true
|
||||
if r.Actor != "oidc:authtest-admin" {
|
||||
t.Errorf("device.edit actor %q, want the session's subject", r.Actor)
|
||||
}
|
||||
}
|
||||
if r.Actor == "anonymous" {
|
||||
t.Errorf("audit row %s has the actor anonymous", r.Action)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("no device.edit audit row in %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeSettings records what the API asked and refuses invalid documents.
|
||||
type fakeSettings struct {
|
||||
mu sync.Mutex
|
||||
yaml string
|
||||
last string
|
||||
}
|
||||
|
||||
func (f *fakeSettings) Current(context.Context) (string, []string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.yaml, []string{"retention"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeSettings) Apply(_ context.Context, actor, y string) (api.SettingsResult, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if strings.Contains(y, "bad") {
|
||||
return api.SettingsResult{}, fmt.Errorf("%w: retention.hot must be at least 1d", api.ErrInvalidSettings)
|
||||
}
|
||||
f.yaml, f.last = y, actor
|
||||
return api.SettingsResult{Applied: []string{"backup"}}, nil
|
||||
}
|
||||
|
||||
func TestSettingsInvalidIs422AndUnchangedValidIsApplied(t *testing.T) {
|
||||
fs := &fakeSettings{yaml: "timezone: UTC\n"}
|
||||
e := newAuthedEnv(t, func(d *api.Deps) { d.Settings = fs })
|
||||
adminSess, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
|
||||
viewerSess, _ := authtest.Mint(t, e.auth, auth.RoleViewer)
|
||||
|
||||
if code := e.call(http.MethodGet, "/api/v1/settings", "", viewerSess).Code; code != http.StatusForbidden {
|
||||
t.Fatalf("a viewer reading settings: %d, want 403", code)
|
||||
}
|
||||
rec := e.call(http.MethodPut, "/api/v1/settings", `{"yaml":"bad: 1"}`, adminSess)
|
||||
if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "retention.hot") {
|
||||
t.Fatalf("invalid settings: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
if fs.yaml != "timezone: UTC\n" {
|
||||
t.Fatalf("an invalid document changed the configuration: %q", fs.yaml)
|
||||
}
|
||||
rec = e.call(http.MethodPut, "/api/v1/settings", `{"yaml":"backup: {at: \"03:00\"}"}`, adminSess)
|
||||
var out struct {
|
||||
Applied []string `json:"applied"`
|
||||
RestartRequired []string `json:"restart_required"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil || rec.Code != http.StatusOK || len(out.Applied) != 1 || out.Applied[0] != "backup" {
|
||||
t.Fatalf("valid settings: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
if fs.last != "oidc:authtest-admin" {
|
||||
t.Fatalf("the change was attributed to %q", fs.last)
|
||||
}
|
||||
}
|
||||
|
||||
func multipartBody(t *testing.T, files map[string]string) (*bytes.Buffer, string) {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
for field, content := range files {
|
||||
fw, err := w.CreateFormFile(field, field+".csv")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = fw.Write([]byte(content))
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &buf, w.FormDataContentType()
|
||||
}
|
||||
|
||||
func (e *authedEnv) importCSV(t *testing.T, c authtest.Creds, files map[string]string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
body, ct := multipartBody(t, files)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/devices/import", body)
|
||||
req.Header.Set("Content-Type", ct)
|
||||
req.AddCookie(c.Cookie)
|
||||
req.Header.Set(auth.CSRFHeader, c.CSRF)
|
||||
rec := httptest.NewRecorder()
|
||||
e.top.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestImportCycleIs422WithPathAndWritesNothingThenAValidImportLands(t *testing.T) {
|
||||
var imported []int64
|
||||
e := newAuthedEnv(t, func(d *api.Deps) { d.OnImported = func(ids []int64) { imported = append(imported, ids...) } })
|
||||
adminSess, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
|
||||
|
||||
cyc := "name,address,parents\na,192.0.2.1,c\nb,192.0.2.2,a\nc,192.0.2.3,b\n"
|
||||
rec := e.importCSV(t, adminSess, map[string]string{"devices": cyc})
|
||||
if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "->") || !strings.Contains(rec.Body.String(), "cycle") {
|
||||
t.Fatalf("cycle import: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
if _, n, err := e.st.ListDevices(context.Background(), store.DeviceFilter{}, store.Page{Limit: 10}); err != nil || n != 0 {
|
||||
t.Fatalf("a rejected import wrote %d devices (%v)", n, err)
|
||||
}
|
||||
|
||||
bad := "name,address\n,192.0.2.1\n"
|
||||
rec = e.importCSV(t, adminSess, map[string]string{"devices": bad})
|
||||
if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "devices.csv row 2") {
|
||||
t.Fatalf("bad row: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
|
||||
ok := "name,address,site,role,parents\ncore,192.0.2.1,dc1,core,\nedge,192.0.2.2,dc1,edge,core\n"
|
||||
rec = e.importCSV(t, adminSess, map[string]string{"devices": ok})
|
||||
var res struct {
|
||||
DevicesCreated int `json:"devices_created"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &res); err != nil || rec.Code != http.StatusOK || res.DevicesCreated != 2 {
|
||||
t.Fatalf("valid import: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
if len(imported) != 2 {
|
||||
t.Fatalf("new devices were not handed to discovery: %v", imported)
|
||||
}
|
||||
viewerSess, _ := authtest.Mint(t, e.auth, auth.RoleViewer)
|
||||
if rec := e.importCSV(t, viewerSess, map[string]string{"devices": ok}); rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("a viewer importing: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPortDescriptionImportSetsTheOperatorOwnedFieldAndSearchFindsIt(t *testing.T) {
|
||||
e := newAuthedEnv(t, nil)
|
||||
adminSess, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
|
||||
d := e.device(t, "sw1", "dc1", "edge")
|
||||
ifc := e.iface(t, d, 1, "Gi1/0/1", "uplink from device")
|
||||
rec := e.importCSV(t, adminSess, map[string]string{"ports": "device,ifname,alias\nsw1,Gi1/0/1,Customer ACME Corp\n"})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("ports import: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
got, err := e.st.GetInterface(context.Background(), ifc.ID)
|
||||
if err != nil || got.Description != "Customer ACME Corp" || got.IfAlias != "uplink from device" {
|
||||
t.Fatalf("interface = %+v (%v): the description must be set and the discovered alias left alone", got, err)
|
||||
}
|
||||
var list struct {
|
||||
Items []struct{ Description, IfAlias string }
|
||||
}
|
||||
rec = e.call(http.MethodGet, "/api/v1/interfaces?q=acme", "", adminSess)
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &list); err != nil || len(list.Items) != 1 || list.Items[0].Description != "Customer ACME Corp" {
|
||||
t.Fatalf("search by description: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
var dv struct {
|
||||
Interfaces []struct{ Description string }
|
||||
}
|
||||
rec = e.call(http.MethodGet, "/api/v1/devices/"+d.Slug, "", adminSess)
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &dv); err != nil || len(dv.Interfaces) != 1 || dv.Interfaces[0].Description != "Customer ACME Corp" {
|
||||
t.Fatalf("device page: %s", rec.Body)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeSeries struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeSeries) DeleteSeries(_ context.Context, st string, m []string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, st+":"+strings.Join(m, ","))
|
||||
return f.err
|
||||
}
|
||||
|
||||
func TestPurgeOnlyAfterThePeriodDeletesSeriesFromBothStoresAndAudits(t *testing.T) {
|
||||
fs := &fakeSeries{}
|
||||
e := newAuthedEnv(t, func(d *api.Deps) { d.Series = fs; d.PurgeAfter = func() time.Duration { return 30 * 24 * time.Hour } })
|
||||
adminSess, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
|
||||
active := e.device(t, "active", "dc1", "core")
|
||||
old := e.device(t, "old", "dc1", "core")
|
||||
e.iface(t, old, 1, "Gi1", "")
|
||||
if err := e.st.RetireDevice(context.Background(), old.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if code := e.call(http.MethodPost, "/api/v1/devices/"+active.Slug+"/purge", "", adminSess).Code; code != http.StatusConflict {
|
||||
t.Fatalf("purging an active device: %d, want 409", code)
|
||||
}
|
||||
if code := e.call(http.MethodPost, "/api/v1/devices/"+old.Slug+"/purge", "", adminSess).Code; code != http.StatusConflict || len(fs.calls) != 0 {
|
||||
t.Fatalf("purging before the period: %d, deletions %v", code, fs.calls)
|
||||
}
|
||||
e.clk.Advance(31 * 24 * time.Hour)
|
||||
adminSess, _ = authtest.Mint(t, e.auth, auth.RoleAdmin) // the first session expired meanwhile
|
||||
fs.err = errors.New("store down")
|
||||
if code := e.call(http.MethodPost, "/api/v1/devices/"+old.Slug+"/purge", "", adminSess).Code; code != http.StatusBadGateway {
|
||||
t.Fatalf("purge with the store down: %d, want 502", code)
|
||||
}
|
||||
if d, _ := e.st.GetDeviceByID(context.Background(), old.ID); d.PurgedAt != nil {
|
||||
t.Fatal("the device was marked purged although its series could not be deleted")
|
||||
}
|
||||
fs.err, fs.calls = nil, nil
|
||||
if code := e.call(http.MethodPost, "/api/v1/devices/"+old.Slug+"/purge", "", adminSess).Code; code != http.StatusNoContent {
|
||||
t.Fatalf("purge: %d", code)
|
||||
}
|
||||
if len(fs.calls) != 2 || !strings.HasPrefix(fs.calls[0], "hot:") || !strings.HasPrefix(fs.calls[1], "long:") ||
|
||||
!strings.Contains(fs.calls[0], `device="`+old.Slug+`"`) || !strings.Contains(fs.calls[0], "iface=~") {
|
||||
t.Fatalf("deletions: %v", fs.calls)
|
||||
}
|
||||
if d, _ := e.st.GetDeviceByID(context.Background(), old.ID); d.PurgedAt == nil {
|
||||
t.Fatal("the device is not marked purged")
|
||||
}
|
||||
}
|
||||
@@ -153,7 +153,7 @@ func (s *server) createMaintenance(ctx context.Context, in *struct{ Body Mainten
|
||||
if err := tx.CreateMaintenanceWindow(ctx, w); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "maintenance.create", "maintenance:"+strconv.FormatInt(w.ID, 10), nil,
|
||||
return tx.AppendAudit(ctx, "anonymous", "maintenance.create", "maintenance:"+strconv.FormatInt(w.ID, 10), nil,
|
||||
map[string]any{"device": in.Body.Device, "group_id": in.Body.GroupID, "ends_at": w.EndsAt, "reason": w.Reason})
|
||||
})
|
||||
if err != nil {
|
||||
@@ -178,7 +178,7 @@ func (s *server) deleteMaintenance(ctx context.Context, in *struct {
|
||||
if err := tx.DeleteMaintenanceWindow(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "maintenance.delete", "maintenance:"+in.ID, nil, nil)
|
||||
return tx.AppendAudit(ctx, "anonymous", "maintenance.delete", "maintenance:"+in.ID, nil, nil)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
)
|
||||
|
||||
// MeView is the caller as nmon sees them.
|
||||
type MeView struct {
|
||||
User string `json:"user" doc:"Stable subject: the IDP subject, the token name, \"breakglass\" or \"dev\"."`
|
||||
Name string `json:"name" doc:"Display name; may be empty."`
|
||||
Role string `json:"role" enum:"viewer,admin"`
|
||||
Via string `json:"via" enum:"session,token,breakglass,dev" doc:"How the caller proved who they are."`
|
||||
CSRF string `json:"csrf" doc:"Send as X-CSRF-Token on every state-changing request made with the session cookie; empty for tokens."`
|
||||
DevOpen bool `json:"dev_open" doc:"Authentication is off: every caller is the admin \"dev\"."`
|
||||
}
|
||||
|
||||
func (s *server) registerMe(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "get-me", Method: http.MethodGet, Path: "/api/v1/me", Tags: []string{"auth"},
|
||||
Summary: "Who am I", Description: "The authenticated caller, their role and the CSRF token to send with writes.",
|
||||
Errors: []int{http.StatusUnauthorized},
|
||||
}, s.getMe)
|
||||
}
|
||||
|
||||
func (s *server) getMe(ctx context.Context, _ *struct{}) (*struct{ Body MeView }, error) {
|
||||
p, ok := auth.FromContext(ctx)
|
||||
if !ok {
|
||||
return nil, huma.Error401Unauthorized("authentication required")
|
||||
}
|
||||
return &struct{ Body MeView }{MeView{
|
||||
User: p.User, Name: p.Name, Role: string(p.Role), Via: string(p.Via), CSRF: p.CSRF, DevOpen: p.Via == auth.ViaDev,
|
||||
}}, nil
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"sort"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
)
|
||||
|
||||
// Route is one operation of the API as the Go 1.22 ServeMux and auth.Policy write it.
|
||||
type Route struct {
|
||||
Method string
|
||||
Pattern string // "PUT /api/v1/devices/{slug}"
|
||||
Path string // "/api/v1/devices/{slug}"
|
||||
}
|
||||
|
||||
// Routes lists every operation huma registers, taken from the same registration that serves them. The
|
||||
// viewer-cannot-mutate test and the auth policy both start from it, so a new route cannot be missed.
|
||||
func Routes() []Route {
|
||||
s := &server{Deps: Deps{}.withDefaults()}
|
||||
api := s.newHuma(http.NewServeMux())
|
||||
s.register(api)
|
||||
var out []Route
|
||||
for path, item := range api.OpenAPI().Paths {
|
||||
for method, op := range map[string]*huma.Operation{
|
||||
http.MethodGet: item.Get, http.MethodPut: item.Put, http.MethodPost: item.Post,
|
||||
http.MethodDelete: item.Delete, http.MethodPatch: item.Patch,
|
||||
} {
|
||||
if op == nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, Route{Method: method, Pattern: method + " " + path, Path: path})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Pattern < out[j].Pattern })
|
||||
return out
|
||||
}
|
||||
|
||||
// adminReads are GET routes only an admin may call: they reveal who can reach nmon or how it is configured.
|
||||
var adminReads = []string{"GET /api/v1/tokens", "GET /api/v1/settings", "GET /api/v1/audit"}
|
||||
|
||||
// viewerPosts are non-GET routes that only read, so a viewer may call them. None today; a query endpoint taking
|
||||
// a body would be listed here.
|
||||
var viewerPosts []string
|
||||
|
||||
// DeclarePolicy states the minimum role of each route that departs from the default (GET and HEAD need viewer,
|
||||
// everything else admin).
|
||||
func DeclarePolicy(p *auth.Policy) {
|
||||
for _, pat := range adminReads {
|
||||
p.Require(pat, auth.RoleAdmin)
|
||||
}
|
||||
for _, pat := range viewerPosts {
|
||||
p.Require(pat, auth.RoleViewer)
|
||||
}
|
||||
}
|
||||
|
||||
// ReadOnlyPosts are the non-GET patterns declared viewer-level, for the viewer-cannot-mutate test to skip.
|
||||
func ReadOnlyPosts() []string { return append([]string(nil), viewerPosts...) }
|
||||
@@ -1,96 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
"github.com/v10networks/nmon/internal/vmsup"
|
||||
)
|
||||
|
||||
// DefaultPurgeAfter is how long a device stays retired before an admin may purge it.
|
||||
const DefaultPurgeAfter = 365 * day
|
||||
|
||||
// SeriesDeleter deletes whole series, by selector, from one metric store ("hot" or "long").
|
||||
type SeriesDeleter interface {
|
||||
DeleteSeries(ctx context.Context, store string, matchers []string) error
|
||||
}
|
||||
|
||||
func (s *server) registerPurge(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "purge-device", Method: http.MethodPost, Path: "/api/v1/devices/{slug}/purge", Tags: []string{"devices"},
|
||||
Summary: "Purge a retired device",
|
||||
Description: "Admin only. Allowed once the device has been retired for the configured period (default one year). Deletes " +
|
||||
"the device's series from both metric stores and marks the inventory row purged; the slug stays reserved.",
|
||||
DefaultStatus: http.StatusNoContent,
|
||||
Errors: []int{http.StatusNotFound, http.StatusConflict, http.StatusServiceUnavailable, http.StatusBadGateway},
|
||||
}, s.purgeDevice)
|
||||
}
|
||||
|
||||
func (s *server) purgeAfter() time.Duration {
|
||||
if s.PurgeAfter != nil {
|
||||
if d := s.PurgeAfter(); d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return DefaultPurgeAfter
|
||||
}
|
||||
|
||||
func (s *server) purgeDevice(ctx context.Context, in *struct {
|
||||
Slug string `path:"slug" doc:"Device slug."`
|
||||
}) (*struct{}, error) {
|
||||
if s.Series == nil {
|
||||
return nil, errNotConfigured("metric store")
|
||||
}
|
||||
d, err := s.Store.GetDeviceBySlug(ctx, in.Slug)
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
if d.PurgedAt != nil {
|
||||
return nil, huma.Error409Conflict("device is already purged")
|
||||
}
|
||||
if d.RetiredAt == nil {
|
||||
return nil, huma.Error409Conflict("only a retired device can be purged")
|
||||
}
|
||||
if wait := s.purgeAfter(); s.Clock.Now().Before(d.RetiredAt.Add(wait)) {
|
||||
return nil, huma.Error409Conflict("device was retired " + d.RetiredAt.UTC().Format(time.RFC3339) +
|
||||
"; it can be purged after " + d.RetiredAt.Add(wait).UTC().Format(time.RFC3339))
|
||||
}
|
||||
matchers, err := s.purgeMatchers(ctx, d)
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
for _, st := range []string{vmsup.StoreHot, vmsup.StoreLong} {
|
||||
if err := s.Series.DeleteSeries(ctx, st, matchers); err != nil {
|
||||
s.Logger.Error("purge: delete series", "device", d.Slug, "store", st, "err", err)
|
||||
return nil, huma.Error502BadGateway("could not delete the device's series from the " + st + " store; nothing was purged")
|
||||
}
|
||||
}
|
||||
if err := s.Store.PurgeDevice(ctx, d.ID); err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
if err := s.Store.AppendAudit(ctx, actorOf(ctx), "device.purge", d.Slug, nil, map[string]any{"retired_at": d.RetiredAt}); err != nil {
|
||||
s.Logger.Warn("audit device.purge", "err", err)
|
||||
}
|
||||
return &struct{}{}, nil
|
||||
}
|
||||
|
||||
// purgeMatchers selects every series of the device: its own (device label) and its interfaces' (iface label).
|
||||
func (s *server) purgeMatchers(ctx context.Context, d *store.Device) ([]string, error) {
|
||||
out := []string{`{device=` + quote(d.Slug) + `}`}
|
||||
ifs, err := s.Store.ListInterfaces(ctx, d.ID, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(ifs) > 0 {
|
||||
ids := make([]string, len(ifs))
|
||||
for i := range ifs {
|
||||
ids[i] = ifs[i].ID
|
||||
}
|
||||
out = append(out, `{iface=~`+quote(regexAlt(ids))+`}`)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -223,10 +223,10 @@ func (s *server) applyResolution(ctx context.Context, r *store.IdentityReview, b
|
||||
}
|
||||
retired = append(retired, id)
|
||||
}
|
||||
if err := tx.ResolveIdentityReview(ctx, r.ID, actorOf(ctx)); err != nil {
|
||||
if err := tx.ResolveIdentityReview(ctx, r.ID, "anonymous"); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.AppendAudit(ctx, actorOf(ctx), "identity_review."+b.Action, "identity_review:"+strconv.FormatInt(r.ID, 10),
|
||||
return tx.AppendAudit(ctx, "anonymous", "identity_review."+b.Action, "identity_review:"+strconv.FormatInt(r.ID, 10),
|
||||
map[string]any{"kind": r.Kind, "interfaces": reviewIfaceIDs(r)}, map[string]any{"kept": b.Keep, "retired": retired})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ func (s *server) repairRollup(ctx context.Context, in *struct{ Body rollup.Repai
|
||||
s.Logger.Error("rollup repair", "err", err)
|
||||
return nil, huma.Error500InternalServerError("rollup repair failed; see the log")
|
||||
}
|
||||
if err := s.Store.AppendAudit(ctx, actorOf(ctx), "rollup.repair", "rollup", nil, in.Body); err != nil {
|
||||
if err := s.Store.AppendAudit(ctx, "anonymous", "rollup.repair", "rollup", nil, in.Body); err != nil {
|
||||
s.Logger.Warn("audit rollup.repair", "err", err)
|
||||
}
|
||||
return &struct{ Body rollup.RepairReport }{rep}, nil
|
||||
|
||||
@@ -47,7 +47,7 @@ func (s *server) putSecret(ctx context.Context, in *struct {
|
||||
if err := w.PutSecret(ctx, in.Name, in.Body.Value); err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
if err := s.Store.AppendAudit(ctx, actorOf(ctx), "secret.put", "secret:"+in.Name, nil, nil); err != nil { // the value is never audited
|
||||
if err := s.Store.AppendAudit(ctx, "anonymous", "secret.put", "secret:"+in.Name, nil, nil); err != nil { // the value is never audited
|
||||
s.Logger.Warn("audit secret.put", "err", err)
|
||||
}
|
||||
return &struct{}{}, nil
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
)
|
||||
|
||||
// ErrInvalidSettings wraps a settings document that failed validation; the running configuration is unchanged.
|
||||
var ErrInvalidSettings = errors.New("invalid settings")
|
||||
|
||||
// SettingsResult says what a settings change did.
|
||||
type SettingsResult struct {
|
||||
// Applied names the sections that took effect in the running process.
|
||||
Applied []string
|
||||
// RestartRequired names the changed sections that only take effect after a restart.
|
||||
RestartRequired []string
|
||||
}
|
||||
|
||||
// SettingsAdmin reads config.yaml and replaces it. Apply validates the whole document first and changes
|
||||
// nothing when it is invalid (the error wraps ErrInvalidSettings); otherwise it writes the file atomically and
|
||||
// reloads the parts that can change while nmon runs.
|
||||
type SettingsAdmin interface {
|
||||
// Current is the configuration as YAML and the sections edited since start that still need a restart.
|
||||
Current(ctx context.Context) (yaml string, restartRequired []string, err error)
|
||||
Apply(ctx context.Context, actor, yaml string) (SettingsResult, error)
|
||||
}
|
||||
|
||||
// SettingsView is the configuration document.
|
||||
type SettingsView struct {
|
||||
YAML string `json:"yaml" doc:"config.yaml. Secrets are never in it; the OIDC client secret is a named secret."`
|
||||
RestartRequired []string `json:"restart_required" doc:"Sections whose saved value differs from what is running and needs a restart."`
|
||||
}
|
||||
|
||||
// SettingsBody replaces the configuration document.
|
||||
type SettingsBody struct {
|
||||
YAML string `json:"yaml" minLength:"1" maxLength:"1048576" doc:"The complete new config.yaml."`
|
||||
}
|
||||
|
||||
// SettingsApplied reports the outcome of PUT /settings.
|
||||
type SettingsApplied struct {
|
||||
YAML string `json:"yaml" doc:"The configuration as saved."`
|
||||
Applied []string `json:"applied" doc:"Sections reloaded in place."`
|
||||
RestartRequired []string `json:"restart_required" doc:"Sections saved but running with the old value until a restart: listen address, TLS, data dir, retention, rules, checks, Proxmox, timezone."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
|
||||
}
|
||||
|
||||
func (s *server) registerSettings(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "get-settings", Method: http.MethodGet, Path: "/api/v1/settings", Tags: []string{"settings"},
|
||||
Summary: "Read the configuration", Description: "Admin only. The effective config.yaml.",
|
||||
Errors: []int{http.StatusServiceUnavailable},
|
||||
}, s.getSettings)
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "put-settings", Method: http.MethodPut, Path: "/api/v1/settings", Tags: []string{"settings"},
|
||||
Summary: "Replace the configuration",
|
||||
Description: "Admin only. Validates the whole document, writes config.yaml atomically and reloads what can change " +
|
||||
"without a restart. An invalid document is a 422 and nothing changes. SIGHUP reloads the file the same way.",
|
||||
Errors: []int{http.StatusUnprocessableEntity, http.StatusServiceUnavailable},
|
||||
}, s.putSettings)
|
||||
}
|
||||
|
||||
func (s *server) getSettings(ctx context.Context, _ *struct{}) (*struct{ Body SettingsView }, error) {
|
||||
if s.Settings == nil {
|
||||
return nil, errNotConfigured("settings")
|
||||
}
|
||||
y, pending, err := s.Settings.Current(ctx)
|
||||
if err != nil {
|
||||
s.Logger.Error("read settings", "err", err)
|
||||
return nil, huma.Error500InternalServerError("could not read the configuration; see the log")
|
||||
}
|
||||
return &struct{ Body SettingsView }{SettingsView{YAML: y, RestartRequired: nonNil(pending)}}, nil
|
||||
}
|
||||
|
||||
func (s *server) putSettings(ctx context.Context, in *struct{ Body SettingsBody }) (*struct{ Body SettingsApplied }, error) {
|
||||
if s.Settings == nil {
|
||||
return nil, errNotConfigured("settings")
|
||||
}
|
||||
res, err := s.Settings.Apply(ctx, actorOf(ctx), in.Body.YAML)
|
||||
if errors.Is(err, ErrInvalidSettings) {
|
||||
return nil, huma.Error422UnprocessableEntity(trimSentinel(err, ErrInvalidSettings))
|
||||
}
|
||||
if err != nil {
|
||||
s.Logger.Error("apply settings", "err", err)
|
||||
return nil, huma.Error500InternalServerError("could not save the configuration; see the log")
|
||||
}
|
||||
y, _, err := s.Settings.Current(ctx)
|
||||
if err != nil {
|
||||
s.Logger.Warn("read back settings", "err", err)
|
||||
}
|
||||
out := SettingsApplied{YAML: y, Applied: nonNil(res.Applied), RestartRequired: nonNil(res.RestartRequired)}
|
||||
return &struct{ Body SettingsApplied }{out}, nil
|
||||
}
|
||||
@@ -9,16 +9,15 @@ import (
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
"github.com/v10networks/nmon/internal/ingest"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// TokenView is a token as listed. The secret and its hash are never part of it.
|
||||
type TokenView struct {
|
||||
ID string `json:"id" doc:"Token id. Viewer and admin tokens have an id such as a7; push tokens have a plain number."`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope" enum:"push,viewer,admin" doc:"push (ingest only), viewer (every read) or admin (everything)."`
|
||||
Scope string `json:"scope" enum:"push" doc:"What the token may do. Only push exists until API tokens arrive."`
|
||||
Devices []string `json:"devices" doc:"Slugs of the devices a push token may write as."`
|
||||
SeriesCap int64 `json:"series_cap" doc:"Most distinct series the sender may keep alive."`
|
||||
ExpiresAt *time.Time `json:"expires_at" nullable:"true"`
|
||||
@@ -39,8 +38,8 @@ type TokenList struct {
|
||||
// TokenBody is the body of POST /tokens.
|
||||
type TokenBody struct {
|
||||
Name string `json:"name" minLength:"1" maxLength:"100"`
|
||||
Scope string `json:"scope,omitempty" enum:"push,viewer,admin" default:"push" doc:"push writes metrics through ingest only; viewer and admin tokens (nma_...) call the API with that role."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
|
||||
Devices []string `json:"devices,omitempty" doc:"Push tokens only, at least one: device slugs the token may push as. With several, the payload's host label picks one."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
|
||||
Scope string `json:"scope,omitempty" enum:"push" default:"push" doc:"Only push is accepted for now."`
|
||||
Devices []string `json:"devices" minItems:"1" doc:"Device slugs the token may push as. With several, the payload's host label picks one."` //nolint:lll // struct tag text is the OpenAPI doc; cannot wrap
|
||||
SeriesCap int64 `json:"series_cap,omitempty" minimum:"0" doc:"Series cap; 0 or absent means the default of 50,000."`
|
||||
ExpiresAt *time.Time `json:"expires_at,omitempty" doc:"After this time the token is rejected."`
|
||||
}
|
||||
@@ -48,23 +47,20 @@ type TokenBody struct {
|
||||
func (s *server) registerTokens(api huma.API) {
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "create-token", Method: http.MethodPost, Path: "/api/v1/tokens", Tags: []string{"tokens"},
|
||||
Summary: "Mint a token", DefaultStatus: http.StatusCreated,
|
||||
Description: "Admin only. The secret is in this response only. A push token (nmp_<id>_<random>) authenticates POST /write, " +
|
||||
"POST /api/v1/write and the Graphite listener, and nothing else. A viewer or admin token (nma_<id>_<random>) " +
|
||||
"authenticates the API with that role in an Authorization: Bearer header.",
|
||||
Summary: "Mint a push token", DefaultStatus: http.StatusCreated,
|
||||
Description: "The secret (nmp_<id>_<random>) is in this response only. A push token authenticates POST /write, " +
|
||||
"POST /api/v1/write and the Graphite listener, and nothing else.",
|
||||
Errors: []int{http.StatusUnprocessableEntity, http.StatusServiceUnavailable},
|
||||
}, s.createToken)
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "list-tokens", Method: http.MethodGet, Path: "/api/v1/tokens", Tags: []string{"tokens"},
|
||||
Summary: "List tokens",
|
||||
Description: "Admin only. Push, viewer and admin tokens; revoked ones stay listed. Secrets and hashes are never returned.",
|
||||
Errors: []int{http.StatusServiceUnavailable},
|
||||
Summary: "List tokens", Description: "Revoked tokens stay listed. Secrets and hashes are never returned.",
|
||||
Errors: []int{http.StatusServiceUnavailable},
|
||||
}, s.listTokens)
|
||||
huma.Register(api, huma.Operation{
|
||||
OperationID: "revoke-token", Method: http.MethodDelete, Path: "/api/v1/tokens/{id}", Tags: []string{"tokens"},
|
||||
Summary: "Revoke a token", Description: "The token is rejected on its next request. Use the id as listed (a7 for an API token).",
|
||||
DefaultStatus: http.StatusNoContent,
|
||||
Errors: []int{http.StatusNotFound, http.StatusServiceUnavailable},
|
||||
Summary: "Revoke a token", Description: "The token is rejected on its next request.", DefaultStatus: http.StatusNoContent,
|
||||
Errors: []int{http.StatusNotFound, http.StatusServiceUnavailable},
|
||||
}, s.revokeToken)
|
||||
}
|
||||
|
||||
@@ -81,13 +77,20 @@ func (s *server) tokenView(ctx context.Context, t *store.PushToken) TokenView {
|
||||
return v
|
||||
}
|
||||
|
||||
// pushDeviceIDs resolves the slugs a push token is bound to; each must be an active device.
|
||||
func (s *server) pushDeviceIDs(ctx context.Context, slugs []string) ([]int64, error) {
|
||||
if len(slugs) == 0 {
|
||||
return nil, huma.Error422UnprocessableEntity("devices: a push token needs at least one device slug")
|
||||
func (s *server) createToken(ctx context.Context, in *struct{ Body TokenBody }) (*struct{ Body TokenCreated }, error) {
|
||||
if s.Tokens == nil {
|
||||
return nil, errNotConfigured("push ingest")
|
||||
}
|
||||
b := in.Body
|
||||
if b.Scope != "" && b.Scope != store.PushScope {
|
||||
return nil, huma.Error422UnprocessableEntity("scope: only \"push\" tokens can be minted here")
|
||||
}
|
||||
name := strings.TrimSpace(b.Name)
|
||||
if name == "" {
|
||||
return nil, huma.Error422UnprocessableEntity("name must not be empty")
|
||||
}
|
||||
var ids []int64
|
||||
for _, slug := range slugs {
|
||||
for _, slug := range b.Devices {
|
||||
d, err := s.Store.GetDeviceBySlug(ctx, slug)
|
||||
if err != nil || d.RetiredAt != nil || d.PurgedAt != nil {
|
||||
if err != nil && !isNotFound(err) {
|
||||
@@ -97,35 +100,16 @@ func (s *server) pushDeviceIDs(ctx context.Context, slugs []string) ([]int64, er
|
||||
}
|
||||
ids = append(ids, d.ID)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (s *server) createToken(ctx context.Context, in *struct{ Body TokenBody }) (*struct{ Body TokenCreated }, error) {
|
||||
b := in.Body
|
||||
name := strings.TrimSpace(b.Name)
|
||||
if name == "" {
|
||||
return nil, huma.Error422UnprocessableEntity("name must not be empty")
|
||||
}
|
||||
if b.ExpiresAt != nil && !b.ExpiresAt.After(s.Clock.Now()) {
|
||||
return nil, huma.Error422UnprocessableEntity("expires_at must be in the future")
|
||||
}
|
||||
if b.Scope == string(auth.RoleViewer) || b.Scope == string(auth.RoleAdmin) {
|
||||
return s.createAPIToken(ctx, name, b)
|
||||
}
|
||||
if s.Tokens == nil {
|
||||
return nil, errNotConfigured("push ingest")
|
||||
}
|
||||
ids, err := s.pushDeviceIDs(ctx, b.Devices)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row, secret, err := s.Tokens.Mint(ctx, ingest.MintOptions{
|
||||
Name: name, DeviceIDs: ids, SeriesCap: int(b.SeriesCap), ExpiresAt: b.ExpiresAt,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
if err := s.Store.AppendAudit(ctx, actorOf(ctx), "token.create", "token:"+strconv.FormatInt(row.ID, 10), nil,
|
||||
if err := s.Store.AppendAudit(ctx, "anonymous", "token.create", "token:"+strconv.FormatInt(row.ID, 10), nil,
|
||||
map[string]any{"name": name, "scope": row.Scope, "devices": b.Devices}); err != nil {
|
||||
s.Logger.Warn("audit token.create", "err", err)
|
||||
}
|
||||
@@ -133,33 +117,23 @@ func (s *server) createToken(ctx context.Context, in *struct{ Body TokenBody })
|
||||
}
|
||||
|
||||
func (s *server) listTokens(ctx context.Context, _ *struct{}) (*struct{ Body TokenList }, error) {
|
||||
if s.Tokens == nil && s.APITokens == nil {
|
||||
return nil, errNotConfigured("tokens")
|
||||
if s.Tokens == nil {
|
||||
return nil, errNotConfigured("push ingest")
|
||||
}
|
||||
out := TokenList{Items: []TokenView{}}
|
||||
if s.Tokens != nil {
|
||||
rows, err := s.Tokens.List(ctx)
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
for i := range rows {
|
||||
out.Items = append(out.Items, s.tokenView(ctx, &rows[i]))
|
||||
}
|
||||
}
|
||||
api, err := s.listAPITokens(ctx)
|
||||
rows, err := s.Tokens.List(ctx)
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
out.Items = append(out.Items, api...)
|
||||
out := TokenList{Items: make([]TokenView, 0, len(rows))}
|
||||
for i := range rows {
|
||||
out.Items = append(out.Items, s.tokenView(ctx, &rows[i]))
|
||||
}
|
||||
return &struct{ Body TokenList }{out}, nil
|
||||
}
|
||||
|
||||
func (s *server) revokeToken(ctx context.Context, in *struct {
|
||||
ID string `path:"id" doc:"Token id."`
|
||||
}) (*struct{}, error) {
|
||||
if rest, ok := strings.CutPrefix(in.ID, apiTokenIDPrefix); ok {
|
||||
return s.revokeAPIToken(ctx, rest)
|
||||
}
|
||||
if s.Tokens == nil {
|
||||
return nil, errNotConfigured("push ingest")
|
||||
}
|
||||
@@ -170,7 +144,7 @@ func (s *server) revokeToken(ctx context.Context, in *struct {
|
||||
if err := s.Tokens.Revoke(ctx, id); err != nil {
|
||||
return nil, s.storeErr(err)
|
||||
}
|
||||
if err := s.Store.AppendAudit(ctx, actorOf(ctx), "token.revoke", "token:"+in.ID, nil, nil); err != nil {
|
||||
if err := s.Store.AppendAudit(ctx, "anonymous", "token.revoke", "token:"+in.ID, nil, nil); err != nil {
|
||||
s.Logger.Warn("audit token.revoke", "err", err)
|
||||
}
|
||||
return &struct{}{}, nil
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
|
||||
"github.com/danielgtaylor/huma/v2"
|
||||
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// apiTokenIDPrefix distinguishes the id of a viewer or admin token from a push token's: the two kinds live in
|
||||
// different tables, so their numbers overlap.
|
||||
const apiTokenIDPrefix = "a"
|
||||
|
||||
// APITokenAdmin mints, lists and revokes viewer and admin API tokens. *auth.Tokens implements it.
|
||||
type APITokenAdmin interface {
|
||||
Mint(ctx context.Context, actor string, o auth.MintOptions) (*store.APIToken, string, error)
|
||||
Revoke(ctx context.Context, actor string, id int64) error
|
||||
List(ctx context.Context) ([]store.APIToken, error)
|
||||
}
|
||||
|
||||
func apiTokenView(t *store.APIToken) TokenView {
|
||||
return TokenView{
|
||||
ID: apiTokenIDPrefix + strconv.FormatInt(t.ID, 10), Name: t.Name, Scope: t.Scope, Devices: []string{},
|
||||
ExpiresAt: utcp(t.ExpiresAt), RevokedAt: utcp(t.RevokedAt),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *server) createAPIToken(ctx context.Context, name string, b TokenBody) (*struct{ Body TokenCreated }, error) {
|
||||
if s.APITokens == nil {
|
||||
return nil, errNotConfigured("API tokens")
|
||||
}
|
||||
if len(b.Devices) > 0 {
|
||||
return nil, huma.Error422UnprocessableEntity("devices: only push tokens are bound to devices")
|
||||
}
|
||||
row, secret, err := s.APITokens.Mint(ctx, actorOf(ctx), auth.MintOptions{Name: name, Scope: auth.Role(b.Scope), ExpiresAt: b.ExpiresAt})
|
||||
if err != nil {
|
||||
return nil, s.storeErr(err) // Mint audits the creation itself
|
||||
}
|
||||
return &struct{ Body TokenCreated }{TokenCreated{TokenView: apiTokenView(row), Secret: secret}}, nil
|
||||
}
|
||||
|
||||
func (s *server) listAPITokens(ctx context.Context) ([]TokenView, error) {
|
||||
if s.APITokens == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.APITokens.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]TokenView, 0, len(rows))
|
||||
for i := range rows {
|
||||
out = append(out, apiTokenView(&rows[i])) // the hash stays in the row
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *server) revokeAPIToken(ctx context.Context, rest string) (*struct{}, error) {
|
||||
if s.APITokens == nil {
|
||||
return nil, errNotConfigured("API tokens")
|
||||
}
|
||||
id, err := idParam(rest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.APITokens.Revoke(ctx, actorOf(ctx), id); err != nil {
|
||||
return nil, s.storeErr(err) // Revoke audits itself
|
||||
}
|
||||
return &struct{}{}, nil
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
package api_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
"github.com/v10networks/nmon/internal/auth/authtest"
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/ingest"
|
||||
"github.com/v10networks/nmon/internal/metrics"
|
||||
"github.com/v10networks/nmon/internal/poller"
|
||||
"github.com/v10networks/nmon/internal/selfmon"
|
||||
"github.com/v10networks/nmon/internal/testutil"
|
||||
)
|
||||
|
||||
// authedEnv is the API as production composes it: the auth middleware over a mux of the API, push ingest and
|
||||
// the /auth/ handlers (internal/app builds the same shape in topHandler).
|
||||
type authedEnv struct {
|
||||
*env
|
||||
auth *auth.Auth
|
||||
top http.Handler
|
||||
}
|
||||
|
||||
func newAuthedEnv(t *testing.T, mutate func(*api.Deps)) *authedEnv {
|
||||
t.Helper()
|
||||
clk := clock.NewFake(time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC))
|
||||
st := testutil.OpenStore(t, clk)
|
||||
a, err := auth.New(context.Background(), auth.Options{
|
||||
Store: st, Clock: clk, IngestPrefixes: []string{"/write", "/api/v1/write"}, Policy: auth.NewPolicy(),
|
||||
BreakGlass: auth.BreakGlassConfig{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api.DeclarePolicy(a.Policy())
|
||||
a.Policy().Public("GET /healthz")
|
||||
e := &env{clk: clk, st: st, q: &querier{body: []byte(`{"status":"success","data":{"resultType":"vector","result":[]}}`)},
|
||||
ref: &refresher{}, reg: selfmon.New()}
|
||||
pushTokens := ingest.NewTokens(st, clk)
|
||||
d := api.Deps{Store: st, Querier: e.q, Poller: e.ref, Clock: clk, Registry: e.reg, DiscoveryTimeout: time.Second,
|
||||
PollerStatus: func() []poller.DeviceStatus { return nil }, Tokens: pushTokens, APITokens: a.Tokens,
|
||||
}
|
||||
if mutate != nil {
|
||||
mutate(&d)
|
||||
}
|
||||
e.h = api.New(d)
|
||||
g, err := ingest.New(ingest.Config{
|
||||
Tokens: pushTokens, Clock: clk, Registry: selfmon.New(),
|
||||
Writer: metrics.WriterFunc(func(context.Context, []metrics.Sample) error { return nil }),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(http.ResponseWriter, *http.Request) {})
|
||||
mux.Handle("/auth/", a.Handler())
|
||||
g.Mount(mux)
|
||||
mux.Handle("/", e.h)
|
||||
return &authedEnv{env: e, auth: a, top: a.Middleware(mux)}
|
||||
}
|
||||
|
||||
var pathParam = regexp.MustCompile(`\{[^}]+\}`)
|
||||
|
||||
// concrete turns a route pattern into a path a request can use.
|
||||
func concrete(path string) string { return pathParam.ReplaceAllString(path, "fuzz") }
|
||||
|
||||
// TestViewerCannotMutate is the M6 gate's core: every state-changing route of the real router refuses a viewer
|
||||
// session (with a valid CSRF token) and a viewer API token with 403, refuses a cookie request without the CSRF
|
||||
// token, and still works for an admin so the check cannot pass by refusing everything.
|
||||
func TestViewerCannotMutate(t *testing.T) {
|
||||
e := newAuthedEnv(t, nil)
|
||||
viewerSess, viewerTok := authtest.Mint(t, e.auth, auth.RoleViewer)
|
||||
adminSess, _ := authtest.Mint(t, e.auth, auth.RoleAdmin)
|
||||
|
||||
readOnly := map[string]bool{}
|
||||
for _, p := range api.ReadOnlyPosts() {
|
||||
readOnly[p] = true
|
||||
}
|
||||
var routes []authtest.Route
|
||||
for _, r := range api.Routes() {
|
||||
routes = append(routes, authtest.Route{Method: r.Method, Path: concrete(r.Path), ReadOnly: readOnly[r.Pattern]})
|
||||
}
|
||||
mutating := 0
|
||||
for _, r := range routes {
|
||||
if !auth.IsSafeMethod(r.Method) && !r.ReadOnly {
|
||||
mutating++
|
||||
}
|
||||
}
|
||||
if mutating < 25 {
|
||||
t.Fatalf("only %d state-changing routes were enumerated from the huma API; the enumeration is broken", mutating)
|
||||
}
|
||||
authtest.CheckViewerCannotMutate(t, e.top, routes, viewerSess, viewerTok, &adminSess)
|
||||
}
|
||||
|
||||
// TestViewerAndPushCredentialsOnIngestAndAuthRoutes covers the routes outside huma: push ingest accepts only a
|
||||
// push token (a viewer session or an API token is refused, 401 or 403), and the /auth/ endpoints that change
|
||||
// state either need no credentials by design (login) or only act on the caller's own session (logout).
|
||||
func TestViewerAndPushCredentialsOnIngestAndAuthRoutes(t *testing.T) {
|
||||
e := newAuthedEnv(t, nil)
|
||||
viewerSess, viewerTok := authtest.Mint(t, e.auth, auth.RoleViewer)
|
||||
for _, path := range []string{"/write", "/api/v1/write?format=influx"} {
|
||||
for name, c := range map[string]authtest.Creds{"viewer session": viewerSess, "viewer token": viewerTok} {
|
||||
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader("m v=1\n"))
|
||||
if c.Token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.Token)
|
||||
}
|
||||
if c.Cookie != nil {
|
||||
req.AddCookie(c.Cookie)
|
||||
req.Header.Set(auth.CSRFHeader, c.CSRF)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
e.top.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusUnauthorized && rec.Code != http.StatusForbidden {
|
||||
t.Errorf("POST %s with a %s: status %d, want 401 or 403", path, name, rec.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Login endpoints are public by design; with nothing configured they say so instead of logging anyone in.
|
||||
for _, path := range []string{"/auth/breakglass", "/auth/login"} {
|
||||
method := http.MethodPost
|
||||
if path == "/auth/login" {
|
||||
method = http.MethodGet
|
||||
}
|
||||
req := httptest.NewRequest(method, path, strings.NewReader(`{"username":"x","password":"y"}`))
|
||||
rec := httptest.NewRecorder()
|
||||
e.top.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Errorf("%s %s with no provider configured: status %d, want 404", method, path, rec.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEveryRouteIsEitherPublicOrNeedsACredential: no declared API route answers an anonymous caller with data.
|
||||
func TestEveryRouteIsEitherPublicOrNeedsACredential(t *testing.T) {
|
||||
e := newAuthedEnv(t, nil)
|
||||
for _, r := range api.Routes() {
|
||||
req := httptest.NewRequest(r.Method, concrete(r.Path), strings.NewReader("{}"))
|
||||
rec := httptest.NewRecorder()
|
||||
e.top.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusUnauthorized {
|
||||
t.Errorf("%s without credentials: status %d, want 401", r.Pattern, rec.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageLoadsRedirectToSignInAndAPICallsGet401(t *testing.T) {
|
||||
e := newAuthedEnv(t, nil)
|
||||
get := func(path string, hdr ...string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
for i := 0; i+1 < len(hdr); i += 2 {
|
||||
req.Header.Set(hdr[i], hdr[i+1])
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
e.top.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
rec := get("/devices/sw1?tab=ifaces")
|
||||
// No IdP is configured in this environment, so the redirect goes to the break-glass prompt.
|
||||
if rec.Code != http.StatusFound || rec.Header().Get("Location") != "/auth/breakglass?return=%2Fdevices%2Fsw1%3Ftab%3Difaces" {
|
||||
t.Fatalf("page load: %d Location %q", rec.Code, rec.Header().Get("Location"))
|
||||
}
|
||||
for _, p := range []string{"/api/v1/devices", "/metrics", "/openapi.json"} {
|
||||
if rec := get(p); rec.Code != http.StatusUnauthorized {
|
||||
t.Errorf("%s without credentials: %d, want 401 (no redirect)", p, rec.Code)
|
||||
}
|
||||
}
|
||||
if rec := get("/", "Authorization", "Bearer nma_1_nope"); rec.Code != http.StatusUnauthorized {
|
||||
t.Errorf("a request with a bad bearer token is an API client: %d, want 401", rec.Code)
|
||||
}
|
||||
if rec := get("/auth/signedout"); rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "Signed out") {
|
||||
t.Errorf("/auth/signedout: %d", rec.Code)
|
||||
}
|
||||
rec = get("/auth/breakglass")
|
||||
if rec.Code != http.StatusNotFound && rec.Code != http.StatusUnauthorized {
|
||||
t.Errorf("/auth/breakglass with no break-glass configured: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/vmsup"
|
||||
"github.com/v10networks/nmon/internal/webui"
|
||||
)
|
||||
|
||||
// deleteTimeout bounds one delete_series call to a metric store.
|
||||
const deleteTimeout = time.Minute
|
||||
|
||||
// apiDeps hands the API everything it needs.
|
||||
func (a *App) apiDeps(s *settings) api.Deps {
|
||||
return api.Deps{
|
||||
Store: a.st, Querier: vmsup.NewQuerier(a.sup), Poller: a.poller, SNMPPort: s.SNMPPort, Logger: a.log,
|
||||
Profiles: a.profiles, PollerStatus: a.poller.Status, AfterDiscover: a.afterDiscover, PollInterval: s.PollInterval,
|
||||
HotRetention: s.cfg.Retention.Hot.D(), FiveMinuteRetention: s.cfg.Retention.FiveMinute.D(), LongRetention: s.cfg.Retention.Long.D(),
|
||||
Reachability: a.alerting.reach, DeviceWarnings: a.alerting.reach.Warnings,
|
||||
Dependencies: depGraph{a.alerting.reach}, OnMaintenanceChange: a.alerting.maint.Refresh,
|
||||
Tokens: a.svc.tokens, Secrets: a.st, Rollup: a.svc.rollup, P95: a.svc.p95, Groups: a.svc.groups, Zones: a.svc.zones,
|
||||
Checks: a.svc.checks, Proxmox: a.svc.proxmox,
|
||||
APITokens: a.auth.Tokens, Settings: a, Backups: a.backups, Stores: a.sup,
|
||||
Series: seriesDeleter{a.sup}, PurgeAfter: func() time.Duration { return a.live.get().PurgeAfter() },
|
||||
OnImported: a.discoverNew,
|
||||
}
|
||||
}
|
||||
|
||||
// discoverNew starts discovery for freshly imported devices.
|
||||
func (a *App) discoverNew(ids []int64) {
|
||||
for _, id := range ids {
|
||||
a.sched.Trigger(id)
|
||||
}
|
||||
}
|
||||
|
||||
// seriesDeleter deletes series from the metric stores through their localhost delete_series API.
|
||||
type seriesDeleter struct{ sup *vmsup.Supervisor }
|
||||
|
||||
// DeleteSeries implements api.SeriesDeleter.
|
||||
func (d seriesDeleter) DeleteSeries(ctx context.Context, store string, matchers []string) error {
|
||||
return vmsup.DeleteSeries(ctx, &http.Client{Timeout: deleteTimeout}, d.sup, store, matchers)
|
||||
}
|
||||
|
||||
// startAPI binds the listen address and serves the API, the UI and push ingest, over TLS unless --insecure-http.
|
||||
func (a *App) startAPI(s *settings) error {
|
||||
ui, err := webui.Handler(webui.Dist(), a.mux(api.New(a.apiDeps(s))))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ui = a.auth.Middleware(ui)
|
||||
ln, err := net.Listen("tcp", s.Listen)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen: %w", err)
|
||||
}
|
||||
a.ln = ln
|
||||
serve := ln
|
||||
if a.tr.cfg != nil {
|
||||
serve = tls.NewListener(ln, a.tr.cfg)
|
||||
}
|
||||
a.srv = &http.Server{Handler: ui, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
|
||||
go func() {
|
||||
if err := a.srv.Serve(serve); !errors.Is(err, http.ErrServerClosed) {
|
||||
a.serveErr <- err
|
||||
}
|
||||
}()
|
||||
a.onStop("http", a.srv.Shutdown)
|
||||
a.log.Info("serving", "addr", ln.Addr().String(), "transport", a.tr.describe())
|
||||
a.startChallenge(s)
|
||||
return nil
|
||||
}
|
||||
|
||||
// startChallenge serves the ACME http-01 challenge. Failing to bind port 80 never stops nmon: certificates
|
||||
// already issued keep working and renewal is retried by autocert.
|
||||
func (a *App) startChallenge(s *settings) {
|
||||
if a.tr.acme == nil {
|
||||
return
|
||||
}
|
||||
addr := s.ACMEListen
|
||||
if addr == "" {
|
||||
addr = DefaultACMEListen
|
||||
}
|
||||
ln, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
a.log.Warn("ACME challenge listener not started; certificates cannot be issued or renewed until "+addr+" can be bound", "err", err)
|
||||
return
|
||||
}
|
||||
srv := &http.Server{Handler: a.tr.acme, ReadHeaderTimeout: 10 * time.Second}
|
||||
go func() { _ = srv.Serve(ln) }()
|
||||
a.onStop("acme challenge", srv.Shutdown)
|
||||
a.log.Info("ACME http-01 challenge listener up", "addr", addr)
|
||||
}
|
||||
|
||||
// stopSchedule ends the nightly backup schedule.
|
||||
func (a *App) stopSchedule() {
|
||||
a.schedMu.Lock()
|
||||
defer a.schedMu.Unlock()
|
||||
if a.stopBackup != nil {
|
||||
a.stopBackup()
|
||||
a.stopBackup = nil
|
||||
}
|
||||
}
|
||||
|
||||
// OnSIGHUP reloads what a running nmon can reload: config.yaml, the TLS certificate pair, and metric stores
|
||||
// that gave up restarting.
|
||||
func (a *App) OnSIGHUP() {
|
||||
if res, err := a.ReloadFile(); err != nil {
|
||||
a.log.Error("SIGHUP: config.yaml is invalid; the running configuration is unchanged", "err", err)
|
||||
} else {
|
||||
a.log.Info("SIGHUP: config reloaded", "applied", res.Applied, "restart_required", res.RestartRequired)
|
||||
}
|
||||
if a.tr.files != nil {
|
||||
if err := a.tr.files.Reload(); err != nil {
|
||||
a.log.Error("SIGHUP: certificate reload failed; the previous certificate keeps serving", "err", err)
|
||||
} else {
|
||||
a.log.Info("SIGHUP: TLS certificate reloaded")
|
||||
}
|
||||
}
|
||||
for _, c := range a.sup.Status() {
|
||||
if !c.Failed {
|
||||
continue
|
||||
}
|
||||
if err := a.sup.Restart(c.Store); err != nil {
|
||||
a.log.Error("SIGHUP: could not resume metric store", "store", c.Store, "err", err)
|
||||
} else {
|
||||
a.log.Info("SIGHUP: resumed metric store", "store", c.Store)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,18 +9,11 @@ import (
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
"github.com/v10networks/nmon/internal/discovery"
|
||||
"github.com/v10networks/nmon/internal/diskguard"
|
||||
"github.com/v10networks/nmon/internal/ingest"
|
||||
"github.com/v10networks/nmon/internal/metrics"
|
||||
"github.com/v10networks/nmon/internal/netpolicy"
|
||||
"github.com/v10networks/nmon/internal/poller"
|
||||
"github.com/v10networks/nmon/internal/profile"
|
||||
@@ -63,16 +56,6 @@ type App struct {
|
||||
|
||||
serveErr chan error // the HTTP server stopped on its own
|
||||
|
||||
dataDir, keyFile string
|
||||
live *liveConfig // the configuration as running; see reload.go
|
||||
tr *transport
|
||||
auth *auth.Auth
|
||||
guard *diskguard.Guard
|
||||
hotW metrics.Writer // a.hot behind the disk guard's new-series admission
|
||||
backups *backupService
|
||||
schedMu sync.Mutex
|
||||
stopBackup func() // ends the nightly backup schedule; nil when none
|
||||
|
||||
// stops holds one step per subsystem that came up, in start order. Stop runs
|
||||
// them in reverse, so a subsystem goes down before the ones it depends on.
|
||||
stops []stopStep
|
||||
@@ -97,19 +80,9 @@ func Run(ctx context.Context, f config.Flags) error {
|
||||
return err
|
||||
}
|
||||
var runErr error
|
||||
hup := make(chan os.Signal, 1)
|
||||
signal.Notify(hup, syscall.SIGHUP)
|
||||
defer signal.Stop(hup)
|
||||
wait:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break wait
|
||||
case runErr = <-a.serveErr:
|
||||
break wait
|
||||
case <-hup:
|
||||
a.OnSIGHUP()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case runErr = <-a.serveErr:
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
defer cancel()
|
||||
@@ -120,10 +93,9 @@ wait:
|
||||
// bounds startup only; call Stop to shut down. A failed Start stops whatever
|
||||
// had already come up.
|
||||
//
|
||||
// Startup order: inventory, metric stores, alerting (built), write queues and the
|
||||
// disk guard, profiles, poller, alerting (started), discovery scheduler, trap receiver,
|
||||
// rollup, p95 and group jobs, Proxmox sync and checks, push ingest (HTTP and Graphite),
|
||||
// authentication, HTTP API, the nightly backup schedule.
|
||||
// Startup order: inventory, metric stores, write queues, profiles, alerting
|
||||
// (built), poller, alerting (started), discovery scheduler, trap receiver,
|
||||
// rollup, p95 and group jobs, Proxmox sync and checks, push ingest (HTTP and Graphite), HTTP API.
|
||||
// Stop reverses it.
|
||||
func Start(ctx context.Context, f config.Flags) (_ *App, err error) {
|
||||
log := slog.Default()
|
||||
@@ -135,8 +107,7 @@ func Start(ctx context.Context, f config.Flags) (_ *App, err error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a := &App{log: log, serveErr: make(chan error, 1), dataDir: s.DataDir, keyFile: s.keyFile, live: newLiveConfig(s.cfg), tr: s.tr}
|
||||
a.backups = &backupService{a: a}
|
||||
a := &App{log: log, serveErr: make(chan error, 1)}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
sctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
@@ -150,14 +121,11 @@ func Start(ctx context.Context, f config.Flags) (_ *App, err error) {
|
||||
if err = a.startMetricStores(ctx, s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = a.newAlerting(s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.startQueues()
|
||||
if err = a.startDisk(s); err != nil {
|
||||
if err = a.loadProfiles(s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = a.loadProfiles(s); err != nil {
|
||||
if err = a.newAlerting(s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.startPoller(s)
|
||||
@@ -177,14 +145,9 @@ func Start(ctx context.Context, f config.Flags) (_ *App, err error) {
|
||||
if err = a.startIngest(s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = a.startAuth(ctx, s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = a.startAPI(s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.setBackupSchedule(s.cfg)
|
||||
a.onStop("backup schedule", func(context.Context) error { a.stopSchedule(); return nil })
|
||||
return a, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
)
|
||||
|
||||
// ingestPrefixes are the paths push ingest authenticates itself (push tokens); the auth middleware leaves them alone.
|
||||
var ingestPrefixes = []string{"/write", "/api/v1/write"}
|
||||
|
||||
// breakGlassAlarmTTL is how long a break-glass alarm stays open after its last raise. A login lingers an hour so
|
||||
// an operator who was not watching still sees it; failures and lockouts clear when the attacker stops.
|
||||
func breakGlassAlarmTTL(kind string) time.Duration {
|
||||
if kind == auth.AlarmBreakGlassLogin {
|
||||
return time.Hour
|
||||
}
|
||||
return 20 * time.Minute
|
||||
}
|
||||
|
||||
// startAuth builds the authentication service and declares the route policy. OIDC needs its client secret, a
|
||||
// named secret set through PUT /api/v1/secrets/{name}; while it is missing OIDC stays off (with a warning) and
|
||||
// break-glass still works, so an admin can sign in and set it.
|
||||
func (a *App) startAuth(ctx context.Context, s *settings) error {
|
||||
ac := s.cfg.Auth
|
||||
opts := auth.Options{
|
||||
Store: a.st, Alarms: kindAlarms{a.alerting.sys, breakGlassAlarmTTL}, Logger: a.log,
|
||||
Session: auth.SessionConfig{
|
||||
InsecureCookies: s.InsecureHTTP, Idle: ac.Session.Idle.D(), Absolute: ac.Session.Absolute.D(),
|
||||
},
|
||||
BreakGlass: auth.BreakGlassConfig{File: ac.BreakGlassFile, Username: ac.BreakGlassUser},
|
||||
TrustedProxies: ac.TrustedProxies, IngestPrefixes: ingestPrefixes, Policy: auth.NewPolicy(),
|
||||
DevOpen: s.mode.devOpen,
|
||||
}
|
||||
oidc, err := a.oidcConfig(ctx, s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts.OIDC = oidc
|
||||
svc, err := auth.New(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("auth: %w", err)
|
||||
}
|
||||
api.DeclarePolicy(svc.Policy())
|
||||
svc.Policy().Public("GET /healthz")
|
||||
a.auth = svc
|
||||
actx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { svc.Run(actx); close(done) }()
|
||||
a.onStop("auth", func(context.Context) error {
|
||||
cancel()
|
||||
<-done
|
||||
return nil
|
||||
})
|
||||
a.logAuth(s, oidc != nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// oidcConfig reads the client secret from the secret store. nil means OIDC is not configured, or its secret
|
||||
// is not stored yet.
|
||||
func (a *App) oidcConfig(ctx context.Context, s *settings) (*auth.OIDCConfig, error) {
|
||||
o := s.cfg.Auth.OIDC
|
||||
if o == nil {
|
||||
return nil, nil
|
||||
}
|
||||
secret, err := a.st.GetSecret(ctx, o.ClientSecretName)
|
||||
switch {
|
||||
case errors.Is(err, store.ErrNotFound):
|
||||
a.log.Warn("OIDC is configured but its client secret is not stored, so OIDC sign-in is OFF; "+
|
||||
"sign in with break-glass, PUT /api/v1/secrets/"+o.ClientSecretName+", and restart", "secret", o.ClientSecretName)
|
||||
return nil, nil
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("read OIDC client secret %q: %w", o.ClientSecretName, err)
|
||||
}
|
||||
return &auth.OIDCConfig{
|
||||
Issuer: o.Issuer, ClientID: o.ClientID, ClientSecret: secret, RedirectURL: o.RedirectURL,
|
||||
GroupsClaim: o.GroupsClaim, AdminGroup: o.AdminGroup,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *App) logAuth(s *settings, oidc bool) {
|
||||
if s.mode.devOpen {
|
||||
warnDevOpen(a.log, s.Listen)
|
||||
return
|
||||
}
|
||||
a.log.Info("authentication on", "oidc", oidc, "break_glass", s.cfg.Auth.BreakGlassFile != "",
|
||||
"trusted_proxies", len(s.cfg.Auth.TrustedProxies))
|
||||
}
|
||||
|
||||
// mux is the whole site behind the auth middleware: health, /auth/, push ingest, the API and the SPA. Only
|
||||
// /auth/*, /healthz and the ingest paths are public. An unauthenticated page load (the SPA, its files, /docs) is
|
||||
// redirected to sign in; an API call gets a 401 problem. The SPA itself has no login page.
|
||||
func (a *App) mux(apiH http.Handler) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
_, _ = w.Write([]byte("ok\n"))
|
||||
})
|
||||
mux.Handle("/auth/", a.auth.Handler())
|
||||
a.ingester.Mount(mux)
|
||||
mux.Handle("/", apiH)
|
||||
return mux
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/alert"
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/backup"
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
backupPrefix = "nmon-"
|
||||
backupSuffix = ".tar"
|
||||
backupAlarmRule = alert.RuleBackupFailed
|
||||
downloadDir = ".downloads"
|
||||
)
|
||||
|
||||
// backupService runs backups inside the serving process, because only it may touch the live stores.
|
||||
type backupService struct {
|
||||
a *App
|
||||
mu sync.Mutex // one backup at a time: two would snapshot the stores twice for nothing
|
||||
log bytes.Buffer
|
||||
}
|
||||
|
||||
func (b *backupService) options(out string, includeKey bool) backup.Options {
|
||||
a := b.a
|
||||
return backup.Options{
|
||||
DataDir: a.dataDir, KeyFile: a.keyFile, IncludeKey: includeKey, OutPath: out, Store: a.st,
|
||||
Stores: backup.StoresFromStatus(a.dataDir, a.sup.Status()), ReserveCheck: a.guard.ReserveCheck,
|
||||
Clock: clock.Real(), Log: &b.log,
|
||||
}
|
||||
}
|
||||
|
||||
// run takes one backup into out.
|
||||
func (b *backupService) run(ctx context.Context, out string, includeKey bool) (*backup.Result, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.log.Reset()
|
||||
res, err := backup.Backup(ctx, b.options(out, includeKey))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.a.log.Info(strings.TrimSpace(b.log.String()), "file", res.Path, "bytes", res.Bytes)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Create implements api.BackupRunner: a backup in a scratch file that the caller streams and deletes.
|
||||
func (b *backupService) Create(ctx context.Context, includeKey bool) (api.BackupFile, error) {
|
||||
dir := filepath.Join(b.a.dataDir, "backups", downloadDir)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return api.BackupFile{}, err
|
||||
}
|
||||
out := filepath.Join(dir, fmt.Sprintf("download-%d%s", clock.Real().Now().UnixNano(), backupSuffix))
|
||||
res, err := b.run(ctx, out, includeKey)
|
||||
if errors.Is(err, backup.ErrReserve) {
|
||||
return api.BackupFile{}, fmt.Errorf("%w: %w", api.ErrBackupRefused, err)
|
||||
}
|
||||
if err != nil {
|
||||
return api.BackupFile{}, err
|
||||
}
|
||||
return api.BackupFile{Path: res.Path, KeyIncluded: res.KeyIncluded, Done: func() { _ = os.Remove(res.Path) }}, nil
|
||||
}
|
||||
|
||||
// scheduled takes the nightly backup into the configured directory and prunes old ones.
|
||||
func (b *backupService) scheduled(ctx context.Context, cfg config.Backup) error {
|
||||
dir := cfg.Dir
|
||||
if dir == "" {
|
||||
dir = filepath.Join(b.a.dataDir, "backups")
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
name := backupPrefix + clock.Real().Now().UTC().Format("20060102-150405") + backupSuffix
|
||||
if _, err := b.run(ctx, filepath.Join(dir, name), false); err != nil {
|
||||
return err
|
||||
}
|
||||
return pruneBackups(dir, cfg.KeepOrDefault())
|
||||
}
|
||||
|
||||
// pruneBackups deletes the oldest nmon-*.tar files in dir beyond keep. Names sort by time.
|
||||
func pruneBackups(dir string, keep int) error {
|
||||
ents, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var names []string
|
||||
for _, e := range ents {
|
||||
if !e.IsDir() && strings.HasPrefix(e.Name(), backupPrefix) && strings.HasSuffix(e.Name(), backupSuffix) {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
var errs []error
|
||||
for len(names) > keep {
|
||||
errs = append(errs, os.Remove(filepath.Join(dir, names[0])))
|
||||
names = names[1:]
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// setBackupSchedule (re)starts the nightly backup for cfg; an empty "at" stops it. Called at start and on reload.
|
||||
func (a *App) setBackupSchedule(cfg config.Config) {
|
||||
a.schedMu.Lock()
|
||||
defer a.schedMu.Unlock()
|
||||
if a.stopBackup != nil {
|
||||
a.stopBackup()
|
||||
a.stopBackup = nil
|
||||
}
|
||||
if cfg.Backup.At == "" {
|
||||
return
|
||||
}
|
||||
loc, err := time.LoadLocation(cfg.Timezone)
|
||||
if err != nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
sch := &backup.Scheduler{
|
||||
Clock: clock.Real(), At: cfg.Backup.At, Loc: loc,
|
||||
Job: func(ctx context.Context) error { return a.backups.scheduled(ctx, cfg.Backup) },
|
||||
OnResult: func(at time.Time, err error) {
|
||||
a.afterScheduledBackup(at, err)
|
||||
},
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { _ = sch.Run(ctx); close(done) }()
|
||||
a.stopBackup = func() { cancel(); <-done }
|
||||
a.log.Info("nightly backup scheduled", "at", cfg.Backup.At, "zone", loc.String(), "keep", cfg.Backup.KeepOrDefault())
|
||||
}
|
||||
|
||||
// afterScheduledBackup logs the result and keeps the backup_failed alarm in step with it.
|
||||
func (a *App) afterScheduledBackup(at time.Time, err error) {
|
||||
ctx := context.Background()
|
||||
if err != nil {
|
||||
a.log.Error("scheduled backup failed", "at", at, "err", err)
|
||||
if rerr := a.alerting.sys.raise(ctx, backupAlarmRule, "nightly", "the nightly backup failed: "+err.Error(), at, 0); rerr != nil {
|
||||
a.log.Warn("raise backup alarm", "err", rerr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if cerr := a.alerting.sys.clear(ctx, backupAlarmRule, "nightly"); cerr != nil {
|
||||
a.log.Warn("clear backup alarm", "err", cerr)
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ func (a *App) startCollectors(s *settings) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("proxmox: %w", err)
|
||||
}
|
||||
run, err := checks.New(s.cfg.Checks, a.policy, clk, a.st, icmp.New(clk), a.hotW, obs, a.log)
|
||||
run, err := checks.New(s.cfg.Checks, a.policy, clk, a.st, icmp.New(clk), a.hot, obs, a.log)
|
||||
if err != nil {
|
||||
return fmt.Errorf("checks: %w", err)
|
||||
}
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/diskguard"
|
||||
)
|
||||
|
||||
// diskAlarmTTL is how long a disk alarm stays open after the condition was last seen: the guard raises on the
|
||||
// edge only, so diskLoop re-raises every minute while the condition holds and the alarm resolves itself after.
|
||||
const (
|
||||
diskAlarmTTL = 3 * time.Minute
|
||||
diskLoopEvery = time.Minute
|
||||
warmWindow = 15 * time.Minute
|
||||
warmTimeout = time.Minute
|
||||
maxWarmSeries = 2_000_000
|
||||
)
|
||||
|
||||
func diskTTL(string) time.Duration { return diskAlarmTTL }
|
||||
|
||||
// startDisk starts the disk guard and puts its admission policy in front of the hot write queue. The poller,
|
||||
// push ingest and the checks write through a.hotW; the rollup writes to the long store and pauses with the guard.
|
||||
func (a *App) startDisk(s *settings) error {
|
||||
g, err := diskguard.New(diskguard.Options{
|
||||
Path: s.DataDir, Alarms: kindAlarms{a.alerting.sys, diskTTL},
|
||||
AlertUsed: s.cfg.Disk.AlertUsed, AdmitUsed: s.cfg.Disk.AdmitUsed, ReserveFree: s.cfg.Disk.ReserveFree,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("disk guard: %w", err)
|
||||
}
|
||||
adm := diskguard.NewAdmission(a.hot, g, 0, nil)
|
||||
a.guard, a.hotW = g, adm
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{}, 2)
|
||||
go func() { g.Run(ctx); done <- struct{}{} }()
|
||||
go func() { a.diskLoop(ctx); done <- struct{}{} }()
|
||||
go a.warmAdmission(ctx, adm)
|
||||
a.onStop("disk guard", func(context.Context) error {
|
||||
cancel()
|
||||
<-done
|
||||
<-done
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// diskLoop keeps the three disk alarms open while their conditions hold.
|
||||
func (a *App) diskLoop(ctx context.Context) {
|
||||
t := clock.Real().NewTicker(diskLoopEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C():
|
||||
}
|
||||
g := a.guard
|
||||
detail := fmt.Sprintf("%.1f%% of %s used", g.Used()*100, a.dataDir)
|
||||
for kind, on := range map[string]bool{
|
||||
diskguard.AlarmNewSeriesHalt: g.RefuseNewSeries(),
|
||||
diskguard.AlarmReserve: g.Throttle(),
|
||||
} {
|
||||
if on {
|
||||
kindAlarms{a.alerting.sys, diskTTL}.Raise(kind, "", detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// warmAdmission tells the admission writer which series the hot store already has, so a restart while the disk is
|
||||
// above 90% does not treat every existing series as new.
|
||||
func (a *App) warmAdmission(ctx context.Context, adm *diskguard.Admission) {
|
||||
wctx, cancel := context.WithTimeout(ctx, warmTimeout)
|
||||
defer cancel()
|
||||
keys, err := hotSeriesKeys(wctx, a.sup.HotURL(), clock.Real().Now())
|
||||
if err != nil {
|
||||
a.log.Warn("could not list existing series for the disk guard; new series are treated as new until written once", "err", err)
|
||||
return
|
||||
}
|
||||
adm.Warm(keys)
|
||||
a.log.Info("disk guard knows the hot store's series", "series", len(keys))
|
||||
}
|
||||
|
||||
// hotSeriesKeys lists the series written recently as metrics.Sample.SeriesKey strings.
|
||||
func hotSeriesKeys(ctx context.Context, base string, now time.Time) ([]string, error) {
|
||||
q := url.Values{
|
||||
"match[]": {`{__name__=~".+"}`}, "start": {fmt.Sprint(now.Add(-warmWindow).Unix())}, "end": {fmt.Sprint(now.Unix())},
|
||||
"limit": {fmt.Sprint(maxWarmSeries)},
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/api/v1/series?"+q.Encode(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("series list: %s", resp.Status)
|
||||
}
|
||||
var body struct {
|
||||
Data []map[string]string `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, len(body.Data))
|
||||
for _, m := range body.Data {
|
||||
keys = append(keys, seriesKeyOf(m))
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// seriesKeyOf builds the key metrics.Sample.SeriesKey would for these labels (sorted by name).
|
||||
func seriesKeyOf(m map[string]string) string {
|
||||
names := make([]string, 0, len(m))
|
||||
for n := range m {
|
||||
if n != "__name__" {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
var b strings.Builder
|
||||
b.WriteString(m["__name__"])
|
||||
for _, n := range names {
|
||||
b.WriteByte(0)
|
||||
b.WriteString(n + "=" + m[n])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
@@ -16,7 +17,7 @@ func (a *App) startIngest(s *settings) error {
|
||||
clk := clock.Real()
|
||||
tokens := ingest.NewTokens(a.st, clk)
|
||||
g, err := ingest.New(ingest.Config{
|
||||
Tokens: tokens, Writer: a.hotW, Throttle: a.guard.Throttle, Clock: clk, Observer: a.alerting.reach, Alarms: ingestAlarms{a.alerting.sys},
|
||||
Tokens: tokens, Writer: a.hot, Clock: clk, Observer: a.alerting.reach, Alarms: ingestAlarms{a.alerting.sys},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ingest: %w", err)
|
||||
@@ -43,3 +44,11 @@ func (a *App) logGraphiteDown(addr string, err error) {
|
||||
}
|
||||
a.log.Warn("graphite listener not started; HTTP push is unaffected", "addr", addr, "err", err)
|
||||
}
|
||||
|
||||
// mountIngest puts the push routes in front of the API handler on one mux.
|
||||
func (a *App) mountIngest(api http.Handler) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
a.ingester.Mount(mux)
|
||||
mux.Handle("/", api)
|
||||
return mux
|
||||
}
|
||||
|
||||
@@ -1,339 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"io"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
)
|
||||
|
||||
// startWith starts a full server over external fake metric stores with config.yaml preset.
|
||||
func startWith(t *testing.T, mut func(*config.Flags), configYAML string) (*App, config.Flags) {
|
||||
t.Helper()
|
||||
f := flagsFor(t, fakeMetricStore(t).URL)
|
||||
if mut != nil {
|
||||
mut(&f)
|
||||
}
|
||||
if configYAML != "" {
|
||||
if err := os.MkdirAll(f.DataDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(config.File(f.DataDir), []byte(configYAML), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
a, err := Start(context.Background(), f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { stopApp(t, a) })
|
||||
return a, f
|
||||
}
|
||||
|
||||
func getBody(t *testing.T, c *http.Client, url string) (int, string) {
|
||||
t.Helper()
|
||||
resp, err := c.Get(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, string(b)
|
||||
}
|
||||
|
||||
func breakGlassFile(t *testing.T) string {
|
||||
t.Helper()
|
||||
h, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := filepath.Join(t.TempDir(), "bg")
|
||||
if err := os.WriteFile(p, h, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestDevOpenModeIsAdminDevWithInsecureHTTPAndNoSignInConfigured(t *testing.T) {
|
||||
a, _ := startWith(t, nil, "")
|
||||
code, body := getBody(t, http.DefaultClient, "http://"+a.Addr()+"/api/v1/me")
|
||||
var me struct {
|
||||
User, Role string
|
||||
DevOpen bool `json:"dev_open"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &me); err != nil || code != http.StatusOK || me.User != "dev" || me.Role != "admin" || !me.DevOpen {
|
||||
t.Fatalf("GET /api/v1/me in dev-open mode: %d %s", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredSignInMeansPagesRedirectAndAPIRoutesNeedACredentialWhileHealthAndSignedOutDoNot(t *testing.T) {
|
||||
a, _ := startWith(t, nil, "auth:\n break_glass_file: "+breakGlassFile(t)+"\n")
|
||||
base := "http://" + a.Addr()
|
||||
c := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
for path, want := range map[string]int{
|
||||
"/healthz": http.StatusOK, "/auth/signedout": http.StatusOK,
|
||||
"/": http.StatusFound, "/devices/x": http.StatusFound, "/assets/app.js": http.StatusFound, "/docs": http.StatusFound,
|
||||
"/api/v1/devices": http.StatusUnauthorized, "/api/v1/me": http.StatusUnauthorized, "/metrics": http.StatusUnauthorized,
|
||||
"/auth/session": http.StatusUnauthorized, "/api/v1/settings": http.StatusUnauthorized, "/openapi.json": http.StatusUnauthorized,
|
||||
} {
|
||||
if code, body := getBody(t, c, base+path); code != want {
|
||||
t.Errorf("GET %s = %d, want %d: %.100s", path, code, want, body)
|
||||
}
|
||||
}
|
||||
resp, err := c.Get(base + "/devices/x?tab=1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if loc := resp.Header.Get("Location"); loc != "/auth/breakglass?return=%2Fdevices%2Fx%3Ftab%3D1" {
|
||||
t.Fatalf("redirect to %q", loc)
|
||||
}
|
||||
// The break-glass prompt: a Basic challenge, then a session and a redirect to the page asked for.
|
||||
type answer struct {
|
||||
code int
|
||||
header http.Header
|
||||
cookies int
|
||||
}
|
||||
do := func(user string) answer {
|
||||
req, _ := http.NewRequest(http.MethodGet, base+"/auth/breakglass?return=%2Fdevices%2Fx", nil)
|
||||
if user != "" {
|
||||
req.SetBasicAuth(user, "pw")
|
||||
}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
return answer{resp.StatusCode, resp.Header, len(resp.Cookies())}
|
||||
}
|
||||
if a := do(""); a.code != http.StatusUnauthorized || !strings.HasPrefix(a.header.Get("WWW-Authenticate"), "Basic") {
|
||||
t.Fatalf("break-glass without credentials: %d %v", a.code, a.header)
|
||||
}
|
||||
if a := do("breakglass"); a.code != http.StatusFound || a.header.Get("Location") != "/devices/x" || a.cookies == 0 {
|
||||
t.Fatalf("break-glass with credentials: %d %v", a.code, a.header)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanTransportAndAuthRefusals(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
f config.Flags
|
||||
auth config.Auth
|
||||
want string
|
||||
}{
|
||||
"no tls and no insecure": {config.Flags{}, config.Auth{}, "--tls-cert"},
|
||||
"half a certificate pair": {config.Flags{TLSCert: "c.pem"}, config.Auth{}, "go together"},
|
||||
"cert and acme": {config.Flags{TLSCert: "c", TLSKey: "k", ACMEHosts: []string{"a.example"}}, config.Auth{}, "not both"},
|
||||
"insecure with tls": {config.Flags{InsecureHTTP: true, ACMEHosts: []string{"a.example"}}, config.Auth{}, "cannot be combined"},
|
||||
"unreadable certificate": {config.Flags{TLSCert: "/nonexistent/c", TLSKey: "/nonexistent/k"}, config.Auth{}, "tls: load"},
|
||||
"acme needs a host": {config.Flags{ACMEHosts: []string{}}, config.Auth{}, "--tls-cert"},
|
||||
"tls and no sign-in method": {config.Flags{ACMEHosts: []string{"a.example"}}, config.Auth{}, "no way to sign in"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
tr, err := planTransport(&tc.f, t.TempDir())
|
||||
if err == nil {
|
||||
_, err = planAuth(tr, tc.auth)
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("error %v, want one mentioning %q", err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
tr, err := planTransport(&config.Flags{InsecureHTTP: true}, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, err := planAuth(tr, config.Auth{}); err != nil || !m.devOpen {
|
||||
t.Fatalf("insecure-http with no sign-in must be dev-open: %+v %v", m, err)
|
||||
}
|
||||
if m, err := planAuth(tr, config.Auth{BreakGlassFile: "x"}); err != nil || m.devOpen {
|
||||
t.Fatalf("a configured sign-in turns dev-open off: %+v %v", m, err)
|
||||
}
|
||||
}
|
||||
|
||||
// selfSigned writes a certificate for 127.0.0.1 and returns its files and pool.
|
||||
func selfSigned(t *testing.T, serial int64) (certFile, keyFile string, pool *x509.CertPool) {
|
||||
t.Helper()
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmpl := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(serial), Subject: pkix.Name{CommonName: "nmon test"},
|
||||
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, KeyUsage: x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true, IsCA: true,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kb, err := x509.MarshalECPrivateKey(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
certFile, keyFile = filepath.Join(dir, "c.pem"), filepath.Join(dir, "k.pem")
|
||||
if err := os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: kb}), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cert, _ := x509.ParseCertificate(der)
|
||||
pool = x509.NewCertPool()
|
||||
pool.AddCert(cert)
|
||||
return certFile, keyFile, pool
|
||||
}
|
||||
|
||||
func TestServesHTTPSWithTheProvidedCertificateAndReloadsItOnSIGHUP(t *testing.T) {
|
||||
certFile, keyFile, pool := selfSigned(t, 1)
|
||||
a, _ := startWith(t, func(f *config.Flags) { f.InsecureHTTP, f.TLSCert, f.TLSKey = false, certFile, keyFile },
|
||||
"auth:\n break_glass_file: "+breakGlassFile(t)+"\n")
|
||||
c := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}}
|
||||
if code, _ := getBody(t, c, "https://"+a.Addr()+"/healthz"); code != http.StatusOK {
|
||||
t.Fatalf("https /healthz = %d", code)
|
||||
}
|
||||
if resp, err := http.Get("http://" + a.Addr() + "/healthz"); err == nil { // Go answers 400 to plain HTTP on a TLS port
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
t.Fatal("plain HTTP was served on the TLS port")
|
||||
}
|
||||
}
|
||||
// A new pair on disk is served after SIGHUP, with no dropped listener.
|
||||
newCert, newKey, newPool := selfSigned(t, 2)
|
||||
copyFile(t, newCert, certFile)
|
||||
copyFile(t, newKey, keyFile)
|
||||
a.OnSIGHUP()
|
||||
c2 := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: newPool, MinVersion: tls.VersionTLS12}}}
|
||||
if code, _ := getBody(t, c2, "https://"+a.Addr()+"/healthz"); code != http.StatusOK {
|
||||
t.Fatalf("after SIGHUP the new certificate is not served: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func copyFile(t *testing.T, from, to string) {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(from)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(to, b, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSettingsApplyRejectsInvalidWithoutChangeAndReloadsWhatCanChange(t *testing.T) {
|
||||
a, f := startWith(t, nil, "")
|
||||
ctx := context.Background()
|
||||
before, err := os.ReadFile(config.File(f.DataDir))
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, bad := range []string{"retention: {hot: 1h}\n", "bogus: 1\n", "backup: {at: \"25:00\"}\n", "auth: {oidc: {issuer: x}}\n",
|
||||
"alerting:\n receivers:\n - {name: a, type: carrier-pigeon}\n", "disk: {alert_used: 0.95, admit_used: 0.9}\n"} {
|
||||
_, err := a.Apply(ctx, "tester", bad)
|
||||
if !errors.Is(err, api.ErrInvalidSettings) {
|
||||
t.Errorf("%q: error %v, want ErrInvalidSettings", bad, err)
|
||||
}
|
||||
}
|
||||
after, _ := os.ReadFile(config.File(f.DataDir))
|
||||
if string(before) != string(after) {
|
||||
t.Fatalf("an invalid document changed config.yaml:\n%s", after)
|
||||
}
|
||||
if err := a.policy.Check("127.0.0.1"); err == nil {
|
||||
t.Fatal("loopback must be denied by default")
|
||||
}
|
||||
|
||||
res, err := a.Apply(ctx, "tester", "alerting:\n outbound_allow: [\"127.0.0.1\"]\nbackup: {at: \"03:30\", keep: 2}\ndisk: {alert_used: 0.7}\n"+
|
||||
"retention: {hot: 40d, fiveminute: 120d, long: 2y}\n")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := "alerting.outbound_allow backup disk"; strings.Join(res.Applied, " ") != want {
|
||||
t.Fatalf("applied %v, want %q", res.Applied, want)
|
||||
}
|
||||
if strings.Join(res.RestartRequired, " ") != "retention" {
|
||||
t.Fatalf("restart_required %v: retention is read once at start", res.RestartRequired)
|
||||
}
|
||||
if err := a.policy.Check("127.0.0.1"); err != nil {
|
||||
t.Fatalf("the reloaded allow-list is not in effect: %v", err)
|
||||
}
|
||||
if got, err := config.Load(f.DataDir); err != nil || got.Backup.At != "03:30" {
|
||||
t.Fatalf("config.yaml after apply: %+v %v", got.Backup, err)
|
||||
}
|
||||
if a.stopBackup == nil {
|
||||
t.Fatal("the nightly backup schedule was not started by the reload")
|
||||
}
|
||||
if _, restart, _ := a.Current(ctx); strings.Join(restart, " ") != "retention" {
|
||||
t.Fatalf("Current reports restart_required %v", restart)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSIGHUPWithABrokenFileKeepsTheRunningConfiguration(t *testing.T) {
|
||||
a, f := startWith(t, nil, "alerting:\n outbound_allow: [\"127.0.0.1\"]\n")
|
||||
if err := a.policy.Check("127.0.0.1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(config.File(f.DataDir), []byte("retention: {hot: 1h}\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.OnSIGHUP()
|
||||
if err := a.policy.Check("127.0.0.1"); err != nil {
|
||||
t.Fatalf("a failed reload changed the running allow-list: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(config.File(f.DataDir), []byte("timezone: UTC\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.OnSIGHUP()
|
||||
if err := a.policy.Check("127.0.0.1"); err == nil {
|
||||
t.Fatal("a valid reload that removes the allow entry must take effect")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneBackupsKeepsTheNewestAndOnlyTouchesNmonArchives(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, n := range []string{"nmon-20261020-020000.tar", "nmon-20261021-020000.tar", "nmon-20261022-020000.tar", "nmon-20261023-020000.tar",
|
||||
"notes.txt", "other-20260101.tar"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, n), []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := pruneBackups(dir, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ents, _ := os.ReadDir(dir)
|
||||
var got []string
|
||||
for _, e := range ents {
|
||||
got = append(got, e.Name())
|
||||
}
|
||||
want := "nmon-20261022-020000.tar nmon-20261023-020000.tar notes.txt other-20260101.tar"
|
||||
if strings.Join(got, " ") != want {
|
||||
t.Fatalf("after pruning: %v, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesKeyMatchesMetricsSampleSeriesKey(t *testing.T) {
|
||||
got := seriesKeyOf(map[string]string{"__name__": "ifHCInOctets", "iface": "01J", "device": "sw1"})
|
||||
if want := "ifHCInOctets\x00device=sw1\x00iface=01J"; got != want {
|
||||
t.Fatalf("%q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/alert"
|
||||
@@ -21,7 +20,6 @@ import (
|
||||
// and resolved notifications of one incident, and every retry, carry the same one.
|
||||
type notifier struct {
|
||||
q *notify.Queue
|
||||
mu sync.RWMutex // guards receivers, which a config reload replaces
|
||||
receivers []string
|
||||
st store.Store
|
||||
devices *ttlCache[string, store.Device]
|
||||
@@ -32,26 +30,10 @@ func newNotifier(
|
||||
q *notify.Queue, rs []notify.Receiver, st store.Store, devices *ttlCache[string, store.Device], log *slog.Logger,
|
||||
) *notifier {
|
||||
n := ¬ifier{q: q, st: st, devices: devices, log: log}
|
||||
n.setReceivers(rs)
|
||||
return n
|
||||
}
|
||||
|
||||
// setReceivers replaces the receivers every notification goes to.
|
||||
func (n *notifier) setReceivers(rs []notify.Receiver) {
|
||||
names := make([]string, 0, len(rs))
|
||||
for _, r := range rs {
|
||||
names = append(names, r.Name)
|
||||
n.receivers = append(n.receivers, r.Name)
|
||||
}
|
||||
n.mu.Lock()
|
||||
n.receivers = names
|
||||
n.mu.Unlock()
|
||||
}
|
||||
|
||||
// names returns the current receivers.
|
||||
func (n *notifier) names() []string {
|
||||
n.mu.RLock()
|
||||
defer n.mu.RUnlock()
|
||||
return n.receivers
|
||||
return n
|
||||
}
|
||||
|
||||
// enqueue sends n to every receiver. A failure on one receiver is returned
|
||||
@@ -60,7 +42,7 @@ func (n *notifier) names() []string {
|
||||
// and therefore deduplicable.
|
||||
func (n *notifier) enqueue(ctx context.Context, note notify.Notification) error {
|
||||
var errs []error
|
||||
for _, r := range n.names() {
|
||||
for _, r := range n.receivers {
|
||||
if _, err := n.q.Enqueue(ctx, r, note); err != nil {
|
||||
errs = append(errs, fmt.Errorf("receiver %s: %w", r, err))
|
||||
}
|
||||
@@ -88,7 +70,7 @@ type reachNotifier struct{ *notifier }
|
||||
|
||||
// Notify implements reach.Notifier.
|
||||
func (r reachNotifier) Notify(ctx context.Context, t reach.Transition) error {
|
||||
if len(r.names()) == 0 {
|
||||
if len(r.receivers) == 0 {
|
||||
return nil
|
||||
}
|
||||
note := notify.Notification{
|
||||
@@ -109,7 +91,7 @@ type alertNotifier struct{ *notifier }
|
||||
|
||||
// Notify implements alert.Notifier.
|
||||
func (a alertNotifier) Notify(ctx context.Context, t alert.Transition) error {
|
||||
if len(a.names()) == 0 {
|
||||
if len(a.receivers) == 0 {
|
||||
return nil
|
||||
}
|
||||
note := notify.Notification{
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/v10networks/nmon/internal/alertcfg"
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
"github.com/v10networks/nmon/internal/netpolicy"
|
||||
)
|
||||
|
||||
// Section names reported by a settings change.
|
||||
const (
|
||||
secReceivers = "alerting.receivers"
|
||||
secOutbound = "alerting.outbound_allow"
|
||||
secProxies = "auth.trusted_proxies"
|
||||
secBackup = "backup"
|
||||
secDisk = "disk"
|
||||
secPurge = "retired_purge_after"
|
||||
|
||||
secRules = "alerting.rules"
|
||||
secAuth = "auth"
|
||||
secRetention = "retention"
|
||||
secTimezone = "timezone"
|
||||
secSites = "sites"
|
||||
secProxmox = "proxmox"
|
||||
secChecks = "checks"
|
||||
)
|
||||
|
||||
// liveConfig is the configuration as the running process sees it. running is what the restart-only parts were
|
||||
// built from at start; current is the last one applied (or saved).
|
||||
type liveConfig struct {
|
||||
mu sync.Mutex
|
||||
running config.Config
|
||||
current config.Config
|
||||
}
|
||||
|
||||
func newLiveConfig(c config.Config) *liveConfig { return &liveConfig{running: c, current: c} }
|
||||
|
||||
func (l *liveConfig) get() config.Config {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.current
|
||||
}
|
||||
|
||||
// restartSections are the parts of the configuration that are read once at start. They are compared on the
|
||||
// resolved values so that formatting differences in the YAML do not count as a change.
|
||||
func restartSections(old, cur config.Config) []string {
|
||||
var out []string
|
||||
add := func(changed bool, name string) {
|
||||
if changed {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
oldRules, errOld := alertcfg.Resolve(old.Alerting)
|
||||
curRules, errCur := alertcfg.Resolve(cur.Alerting)
|
||||
add(errOld != nil || errCur != nil || !reflect.DeepEqual(oldRules.Rules, curRules.Rules) || oldRules.Interval != curRules.Interval,
|
||||
secRules)
|
||||
add(!reflect.DeepEqual(authExceptProxies(old.Auth), authExceptProxies(cur.Auth)), secAuth)
|
||||
add(old.Retention != cur.Retention, secRetention)
|
||||
add(old.Timezone != cur.Timezone, secTimezone)
|
||||
add(!reflect.DeepEqual(old.Sites, cur.Sites), secSites)
|
||||
add(!reflect.DeepEqual(old.Proxmox, cur.Proxmox), secProxmox)
|
||||
add(!reflect.DeepEqual(old.Checks, cur.Checks), secChecks)
|
||||
return out
|
||||
}
|
||||
|
||||
func authExceptProxies(a config.Auth) config.Auth {
|
||||
a.TrustedProxies = nil
|
||||
return a
|
||||
}
|
||||
|
||||
// Current implements api.SettingsAdmin.
|
||||
func (a *App) Current(context.Context) (string, []string, error) {
|
||||
cur := a.live.get()
|
||||
b, err := yaml.Marshal(cur)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
a.live.mu.Lock()
|
||||
running := a.live.running
|
||||
a.live.mu.Unlock()
|
||||
return string(b), restartSections(running, cur), nil
|
||||
}
|
||||
|
||||
// Apply implements api.SettingsAdmin: validate everything first, then save, then reload in place.
|
||||
func (a *App) Apply(ctx context.Context, actor, text string) (api.SettingsResult, error) {
|
||||
cfg, err := decodeStrict(text)
|
||||
if err != nil {
|
||||
return api.SettingsResult{}, fmt.Errorf("%w: %w", api.ErrInvalidSettings, err)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return api.SettingsResult{}, fmt.Errorf("%w: %w", api.ErrInvalidSettings, err)
|
||||
}
|
||||
if _, err := alertcfg.Resolve(cfg.Alerting); err != nil {
|
||||
return api.SettingsResult{}, fmt.Errorf("%w: %w", api.ErrInvalidSettings, err)
|
||||
}
|
||||
if err := config.Save(a.dataDir, cfg); err != nil {
|
||||
return api.SettingsResult{}, err
|
||||
}
|
||||
res := a.reloadWith(cfg)
|
||||
if err := a.st.AppendAudit(ctx, actor, "settings.update", "config.yaml", nil,
|
||||
map[string][]string{"applied": res.Applied, "restart_required": res.RestartRequired}); err != nil {
|
||||
a.log.Warn("audit settings.update", "err", err)
|
||||
}
|
||||
a.log.Info("settings saved", "by", actor, "applied", res.Applied, "restart_required", res.RestartRequired)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// decodeStrict parses a config document over the defaults, rejecting keys nmon does not know.
|
||||
func decodeStrict(text string) (config.Config, error) {
|
||||
cfg := config.Default()
|
||||
dec := yaml.NewDecoder(bytes.NewReader([]byte(text)))
|
||||
dec.KnownFields(true)
|
||||
if err := dec.Decode(&cfg); err != nil && !errors.Is(err, io.EOF) {
|
||||
return cfg, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// ReloadFile rereads config.yaml (SIGHUP). An invalid file changes nothing and returns the error.
|
||||
func (a *App) ReloadFile() (api.SettingsResult, error) {
|
||||
cfg, err := config.Load(a.dataDir)
|
||||
if err != nil {
|
||||
return api.SettingsResult{}, err
|
||||
}
|
||||
if _, err := alertcfg.Resolve(cfg.Alerting); err != nil {
|
||||
return api.SettingsResult{}, fmt.Errorf("config.yaml: %w", err)
|
||||
}
|
||||
return a.reloadWith(cfg), nil
|
||||
}
|
||||
|
||||
// reloadWith applies the parts of cfg that can change while running. Everything it calls has been validated.
|
||||
func (a *App) reloadWith(cfg config.Config) api.SettingsResult {
|
||||
old := a.live.get()
|
||||
al, _ := alertcfg.Resolve(cfg.Alerting) // validated by the caller
|
||||
oldAl, _ := alertcfg.Resolve(old.Alerting)
|
||||
var applied []string
|
||||
apply := func(changed bool, name string, fn func() error) {
|
||||
if !changed {
|
||||
return
|
||||
}
|
||||
if err := fn(); err != nil {
|
||||
a.log.Error("reload: could not apply "+name, "err", err)
|
||||
return
|
||||
}
|
||||
applied = append(applied, name)
|
||||
}
|
||||
apply(!reflect.DeepEqual(oldAl.Receivers, al.Receivers), secReceivers, func() error { return a.setReceivers(al) })
|
||||
apply(!reflect.DeepEqual(oldAl.OutboundAllow, al.OutboundAllow), secOutbound, func() error {
|
||||
return a.policy.Reload(netpolicy.Config{Allow: al.OutboundAllow})
|
||||
})
|
||||
apply(!reflect.DeepEqual(old.Auth.TrustedProxies, cfg.Auth.TrustedProxies), secProxies, func() error {
|
||||
return a.auth.SetTrustedProxies(cfg.Auth.TrustedProxies)
|
||||
})
|
||||
apply(old.Backup != cfg.Backup, secBackup, func() error { a.setBackupSchedule(cfg); return nil })
|
||||
apply(old.Disk != cfg.Disk, secDisk, func() error {
|
||||
a.guard.SetThresholds(cfg.Disk.AlertUsed, cfg.Disk.AdmitUsed, cfg.Disk.ReserveFree)
|
||||
return nil
|
||||
})
|
||||
apply(old.RetiredPurgeAfter != cfg.RetiredPurgeAfter, secPurge, func() error { return nil })
|
||||
a.live.mu.Lock()
|
||||
a.live.current = cfg
|
||||
running := a.live.running
|
||||
a.live.mu.Unlock()
|
||||
return api.SettingsResult{Applied: applied, RestartRequired: restartSections(running, cfg)}
|
||||
}
|
||||
|
||||
// setReceivers swaps the notification receivers: the delivery queue first (it validates), then the fan-out list.
|
||||
func (a *App) setReceivers(al alertcfg.Settings) error {
|
||||
if err := a.alerting.queue.SetReceivers(al.Receivers); err != nil {
|
||||
return err
|
||||
}
|
||||
a.alerting.nt.setReceivers(al.Receivers)
|
||||
return nil
|
||||
}
|
||||
@@ -37,8 +37,7 @@ func (a *App) startRollup(s *settings) error {
|
||||
ls := rollup.NewLongStore(a.sup, q)
|
||||
origin := a.installTime(context.Background(), clk)
|
||||
job, err := rollup.New(rollup.Options{
|
||||
Query: q, Inventory: a.st, State: a.st, Writer: ls, Deleter: ls, Queue: a.hot, Paused: a.guard.PauseRollup, Clock: clk,
|
||||
Logger: a.log, Origin: origin,
|
||||
Query: q, Inventory: a.st, State: a.st, Writer: ls, Deleter: ls, Queue: a.hot, Clock: clk, Logger: a.log, Origin: origin,
|
||||
Gauges: profileGauges(a.profiles), Alarms: ruleAlarms{a.alerting.sys, alert.RuleRollupStalled},
|
||||
HotRetention: s.cfg.Retention.Hot.D(), FiveMinuteRetention: s.cfg.Retention.FiveMinute.D(), LongRetention: s.cfg.Retention.Long.D(),
|
||||
})
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/v10networks/nmon/internal/auth"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
)
|
||||
|
||||
// DefaultACMEListen is where the http-01 challenge is served; the CA connects to port 80.
|
||||
const DefaultACMEListen = ":80"
|
||||
|
||||
// transport is how the API is served: plain HTTP for development, or HTTPS.
|
||||
type transport struct {
|
||||
insecure bool
|
||||
files *auth.CertFiles // provided certificate, reloaded on SIGHUP; nil with ACME or plain HTTP
|
||||
cfg *tls.Config // nil for plain HTTP
|
||||
acme http.Handler // the http-01 challenge handler when ACME is on
|
||||
}
|
||||
|
||||
// planTransport decides how to serve and refuses the combinations that would expose nmon unprotected:
|
||||
// no TLS without --insecure-http, both at once, or half a certificate pair.
|
||||
func planTransport(f *config.Flags, dataDir string) (*transport, error) {
|
||||
provided := f.TLSCert != "" || f.TLSKey != ""
|
||||
acme := len(f.ACMEHosts) > 0
|
||||
switch {
|
||||
case provided && (f.TLSCert == "" || f.TLSKey == ""):
|
||||
return nil, errors.New("--tls-cert and --tls-key go together")
|
||||
case provided && acme:
|
||||
return nil, errors.New("use either --tls-cert/--tls-key or --acme-host, not both")
|
||||
case f.InsecureHTTP && (provided || acme):
|
||||
return nil, errors.New("--insecure-http serves plain HTTP; it cannot be combined with TLS flags")
|
||||
case f.InsecureHTTP:
|
||||
return &transport{insecure: true}, nil
|
||||
case provided:
|
||||
files, err := auth.LoadCertFiles(f.TLSCert, f.TLSKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &transport{files: files, cfg: files.Config()}, nil
|
||||
case acme:
|
||||
cfg, challenge, err := auth.ACME(auth.ACMEOptions{DataDir: dataDir, Hosts: f.ACMEHosts, Email: f.ACMEEmail})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &transport{cfg: cfg, acme: challenge}, nil
|
||||
}
|
||||
return nil, errors.New("nmon serves HTTPS by default and has no certificate: pass --tls-cert and --tls-key, " +
|
||||
"or --acme-host (repeatable) and --acme-email for Let's Encrypt; " +
|
||||
"for development only, --insecure-http serves plain HTTP")
|
||||
}
|
||||
|
||||
// authMode says how callers are authenticated.
|
||||
type authMode struct {
|
||||
// devOpen: no authentication; only allowed together with --insecure-http.
|
||||
devOpen bool
|
||||
}
|
||||
|
||||
// planAuth refuses to serve over TLS with no way to sign in, and turns authentication off only for
|
||||
// --insecure-http with no OIDC and no break-glass file (development mode, DEVIATIONS 31).
|
||||
func planAuth(tr *transport, a config.Auth) (authMode, error) {
|
||||
configured := a.OIDC != nil || a.BreakGlassFile != ""
|
||||
switch {
|
||||
case configured:
|
||||
return authMode{}, nil
|
||||
case tr.insecure:
|
||||
return authMode{devOpen: true}, nil
|
||||
}
|
||||
return authMode{}, errors.New("no way to sign in is configured: set auth.oidc and/or auth.break_glass_file in config.yaml " +
|
||||
"(nmon will not serve over TLS with authentication off)")
|
||||
}
|
||||
|
||||
// warnDevOpen logs, loudly, that every request is an admin.
|
||||
func warnDevOpen(log *slog.Logger, addr string) {
|
||||
log.Warn("AUTHENTICATION IS DISABLED (development mode): every request is the admin \"dev\"; "+
|
||||
"plain HTTP is served on "+addr+"; configure auth.oidc or auth.break_glass_file and use TLS before exposing this port",
|
||||
"mode", "dev-open")
|
||||
}
|
||||
|
||||
// describe is the transport in one startup log line.
|
||||
func (t *transport) describe() string {
|
||||
switch {
|
||||
case t.insecure:
|
||||
return "plain HTTP (--insecure-http)"
|
||||
case t.acme != nil:
|
||||
return "HTTPS with ACME certificates"
|
||||
}
|
||||
return "HTTPS with the provided certificate"
|
||||
}
|
||||
@@ -26,16 +26,18 @@ type settings struct {
|
||||
cfg config.Config // <data-dir>/config.yaml over the defaults
|
||||
// alerting is cfg.Alerting decoded and validated.
|
||||
alerting alertcfg.Settings
|
||||
tr *transport
|
||||
mode authMode
|
||||
}
|
||||
|
||||
func (s *settings) profilesDir() string { return config.ProfilesDir(s.DataDir) }
|
||||
|
||||
// resolveSettings refuses start-up configurations that would put data or access at risk
|
||||
// (no TLS and no --insecure-http, TLS with no way to sign in, the key inside the data dir) and fills in
|
||||
// resolveSettings refuses start-up configurations that would put data at risk
|
||||
// (no --insecure-http, no data dir, the key inside the data dir) and fills in
|
||||
// the defaults. It creates the data dir but never the key.
|
||||
func resolveSettings(f config.Flags) (*settings, error) {
|
||||
if !f.InsecureHTTP {
|
||||
return nil, errors.New("TLS and authentication arrive in M6; " +
|
||||
"until then pass --insecure-http to run without them (development only)")
|
||||
}
|
||||
if f.DataDir == "" {
|
||||
return nil, errors.New("--data-dir is required")
|
||||
}
|
||||
@@ -67,15 +69,7 @@ func resolveSettings(f config.Flags) (*settings, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("config.yaml: %w", err)
|
||||
}
|
||||
tr, err := planTransport(&f, dataDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode, err := planAuth(tr, cfg.Auth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := &settings{Flags: f, keyFile: keyFile, driver: f.DBDriver, dsn: f.DBDSN, cfg: cfg, alerting: al, tr: tr, mode: mode}
|
||||
s := &settings{Flags: f, keyFile: keyFile, driver: f.DBDriver, dsn: f.DBDSN, cfg: cfg, alerting: al}
|
||||
if s.driver == "" {
|
||||
s.driver = store.DriverSQLite
|
||||
}
|
||||
|
||||
@@ -39,13 +39,7 @@ type sysAlarms struct {
|
||||
expires map[string]time.Time // series key -> when an unrefreshed alarm resolves itself
|
||||
}
|
||||
|
||||
// ingestRaise is one queued raise. rule is empty for ingest alarms (rule = "ingest_" + kind); other
|
||||
// subsystems name their rule and how long the alarm stays open without being raised again.
|
||||
type ingestRaise struct {
|
||||
kind, token, detail string
|
||||
rule string
|
||||
ttl time.Duration
|
||||
}
|
||||
type ingestRaise struct{ kind, token, detail string }
|
||||
|
||||
func newSysAlarms(st store.Alerts, clk clock.Clock, nt alert.Notifier, log *slog.Logger) *sysAlarms {
|
||||
return &sysAlarms{st: st, clk: clk, nt: nt, log: log, in: make(chan ingestRaise, sysQueueSize), expires: map[string]time.Time{}}
|
||||
@@ -150,13 +144,9 @@ func (s *sysAlarms) Run(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (s *sysAlarms) handleIngest(ctx context.Context, r ingestRaise) {
|
||||
rule, ttl := r.rule, r.ttl
|
||||
summary := fmt.Sprintf("%s: %s: %s", r.kind, r.token, r.detail)
|
||||
if rule == "" {
|
||||
rule, ttl = alert.RuleIngestPrefix+r.kind, ingestAlarmTTL
|
||||
summary = fmt.Sprintf("push ingest %s tripped for token %q: %s", r.kind, r.token, r.detail)
|
||||
}
|
||||
if err := s.raise(ctx, rule, r.token, summary, time.Time{}, ttl); err != nil {
|
||||
rule := alert.RuleIngestPrefix + r.kind
|
||||
summary := fmt.Sprintf("push ingest %s tripped for token %q: %s", r.kind, r.token, r.detail)
|
||||
if err := s.raise(ctx, rule, r.token, summary, time.Time{}, ingestAlarmTTL); err != nil {
|
||||
s.log.Warn("raise ingest alarm", "rule", rule, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -199,7 +189,7 @@ type ingestAlarms struct{ s *sysAlarms }
|
||||
// Raise implements ingest.Alarms.
|
||||
func (a ingestAlarms) Raise(kind, token, detail string) {
|
||||
select {
|
||||
case a.s.in <- ingestRaise{kind: kind, token: token, detail: detail}:
|
||||
case a.s.in <- ingestRaise{kind, token, detail}:
|
||||
default: // the queue is full of alarms already; the drop counters still tell the story
|
||||
}
|
||||
}
|
||||
@@ -217,18 +207,3 @@ func (r ruleAlarms) Raise(ctx context.Context, key, summary string, since time.T
|
||||
|
||||
// Clear implements p95.Alarms and rollup.Alarms.
|
||||
func (r ruleAlarms) Clear(ctx context.Context, key string) error { return r.s.clear(ctx, r.rule, key) }
|
||||
|
||||
// kindAlarms adapts diskguard.Alarms and auth.Alarms: the alarm kind is the rule name, the subject is the
|
||||
// second argument (an address, a path), and the alarm resolves on its own ttl after the last raise.
|
||||
type kindAlarms struct {
|
||||
s *sysAlarms
|
||||
ttl func(kind string) time.Duration
|
||||
}
|
||||
|
||||
// Raise implements diskguard.Alarms and auth.Alarms. It only queues.
|
||||
func (k kindAlarms) Raise(kind, subject, detail string) {
|
||||
select {
|
||||
case k.s.in <- ingestRaise{kind: kind, token: subject, detail: detail, rule: kind, ttl: k.ttl(kind)}:
|
||||
default: // the queue is full of alarms already
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,12 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/api"
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
"github.com/v10networks/nmon/internal/config"
|
||||
"github.com/v10networks/nmon/internal/discovery"
|
||||
@@ -16,6 +20,7 @@ import (
|
||||
"github.com/v10networks/nmon/internal/store"
|
||||
"github.com/v10networks/nmon/internal/traps"
|
||||
"github.com/v10networks/nmon/internal/vmsup"
|
||||
"github.com/v10networks/nmon/internal/webui"
|
||||
)
|
||||
|
||||
// Each start* method brings one subsystem up and registers, with onStop, how
|
||||
@@ -92,7 +97,7 @@ func (a *App) loadProfiles(s *settings) error {
|
||||
func (a *App) startPoller(s *settings) {
|
||||
dir := s.profilesDir()
|
||||
a.poller = poller.New(poller.Options{
|
||||
Store: a.st, Writer: a.hotW, Interval: s.PollInterval, MaxInflight: s.MaxInflight, SNMPPort: s.SNMPPort, Logger: a.log,
|
||||
Store: a.st, Writer: a.hot, Interval: s.PollInterval, MaxInflight: s.MaxInflight, SNMPPort: s.SNMPPort, Logger: a.log,
|
||||
Profiles: a.profiles,
|
||||
LoadProfiles: func() (*profile.Set, error) { return profile.Load(dir) },
|
||||
Observer: a.alerting.reach, ICMPOff: s.ICMP == config.ICMPOff,
|
||||
@@ -173,6 +178,37 @@ func (a *App) logTrapsDown(addr string, err error) {
|
||||
a.log.Warn("trap receiver not started; polling and discovery are unaffected", "addr", addr, "err", err)
|
||||
}
|
||||
|
||||
// startAPI binds the listen address and serves the HTTP API.
|
||||
func (a *App) startAPI(s *settings) error {
|
||||
handler := api.New(api.Deps{
|
||||
Store: a.st, Querier: vmsup.NewQuerier(a.sup), Poller: a.poller, SNMPPort: s.SNMPPort, Logger: a.log,
|
||||
Profiles: a.profiles, PollerStatus: a.poller.Status, AfterDiscover: a.afterDiscover, PollInterval: s.PollInterval,
|
||||
HotRetention: s.cfg.Retention.Hot.D(), FiveMinuteRetention: s.cfg.Retention.FiveMinute.D(), LongRetention: s.cfg.Retention.Long.D(),
|
||||
Reachability: a.alerting.reach, DeviceWarnings: a.alerting.reach.Warnings,
|
||||
Dependencies: depGraph{a.alerting.reach}, OnMaintenanceChange: a.alerting.maint.Refresh,
|
||||
Tokens: a.svc.tokens, Secrets: a.st, Rollup: a.svc.rollup, P95: a.svc.p95, Groups: a.svc.groups, Zones: a.svc.zones,
|
||||
Checks: a.svc.checks, Proxmox: a.svc.proxmox,
|
||||
})
|
||||
ui, err := webui.Handler(webui.Dist(), a.mountIngest(handler))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ln, err := net.Listen("tcp", s.Listen)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen: %w", err)
|
||||
}
|
||||
a.ln = ln
|
||||
a.srv = &http.Server{Handler: ui, ReadHeaderTimeout: 10 * time.Second, WriteTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
|
||||
go func() {
|
||||
if err := a.srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
|
||||
a.serveErr <- err
|
||||
}
|
||||
}()
|
||||
a.onStop("http", a.srv.Shutdown)
|
||||
a.log.Warn("serving plain HTTP without authentication (--insecure-http); do not expose this port", "addr", ln.Addr().String())
|
||||
return nil
|
||||
}
|
||||
|
||||
// discoverable lists the devices the scheduler should discover.
|
||||
func (a *App) discoverable(ctx context.Context) ([]int64, error) {
|
||||
var ids []int64
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/v10networks/nmon/internal/clock"
|
||||
@@ -30,9 +29,6 @@ type Options struct {
|
||||
IngestPrefixes []string
|
||||
// Policy declares per-route minimum roles; nil means defaults only.
|
||||
Policy *Policy
|
||||
// DevOpen turns authentication off: every request outside ingest is an admin Principal named "dev".
|
||||
// It exists for `serve --insecure-http` with no OIDC or break-glass configured (development only).
|
||||
DevOpen bool
|
||||
// KeyRotateEvery is how often the cookie-signing key rotates; default 30 days.
|
||||
KeyRotateEvery time.Duration
|
||||
}
|
||||
@@ -43,7 +39,7 @@ const DefaultKeyRotateEvery = 30 * 24 * time.Hour
|
||||
// Auth is the assembled authentication service.
|
||||
type Auth struct {
|
||||
opts Options
|
||||
proxies atomic.Pointer[Proxies]
|
||||
proxies *Proxies
|
||||
signer *signer
|
||||
Sessions *Sessions
|
||||
Tokens *Tokens
|
||||
@@ -73,8 +69,7 @@ func New(ctx context.Context, opts Options) (*Auth, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a := &Auth{opts: opts, signer: sg}
|
||||
a.proxies.Store(proxies)
|
||||
a := &Auth{opts: opts, proxies: proxies, signer: sg}
|
||||
if a.opts.Policy == nil {
|
||||
a.opts.Policy = NewPolicy()
|
||||
}
|
||||
@@ -109,27 +104,11 @@ func (a *Auth) build() error {
|
||||
// declareOwnRoutes registers the /auth/ endpoints in the policy: login, callback and break-glass need no
|
||||
// credentials; logout and the session probe need any authenticated user.
|
||||
func (a *Auth) declareOwnRoutes() {
|
||||
a.opts.Policy.Public("GET /auth/login", "GET /auth/callback", "GET /auth/breakglass", "POST /auth/breakglass", "GET /auth/signedout")
|
||||
a.opts.Policy.Public("GET /auth/login", "GET /auth/callback", "POST /auth/breakglass")
|
||||
a.opts.Policy.Require("POST /auth/logout", RoleViewer)
|
||||
a.opts.Policy.Require("GET /auth/session", RoleViewer)
|
||||
}
|
||||
|
||||
// SetTrustedProxies replaces the trusted-proxy list (config reload). An invalid list changes nothing.
|
||||
func (a *Auth) SetTrustedProxies(cidrs []string) error {
|
||||
p, err := ParseProxies(cidrs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.proxies.Store(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DevOpen reports whether authentication is off.
|
||||
func (a *Auth) DevOpen() bool { return a.opts.DevOpen }
|
||||
|
||||
// OIDCEnabled reports whether an identity provider is configured.
|
||||
func (a *Auth) OIDCEnabled() bool { return a.OIDC != nil }
|
||||
|
||||
// Policy returns the route policy, for the API to declare its routes on.
|
||||
func (a *Auth) Policy() *Policy { return a.opts.Policy }
|
||||
|
||||
|
||||