nmon/cmd/nmon-devtools/seed.go

package main

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/v10networks/nmon/internal/profile"
	"github.com/v10networks/nmon/internal/simagent"
)

// seedConfig is the command line of `devtools seed-demo`.
type seedConfig struct {
	api, hotURL string
	days        int
	step        time.Duration
	countersRec string
	envOut      string
	profilesDir string
	workers     int
	hc          *http.Client
}

// seeded is one device after it was added to nmon.
type seeded struct {
	demoDevice
	slug   string
	agent  *simagent.Agent
	ip     string
	ifaces []seedIface
}

type seedIface struct {
	ID         string `json:"id"`
	Name       string `json:"ifname"`
	HC         bool   `json:"has_hc_counters"`
	Excluded   bool   `json:"excluded"`
	Unreliable bool   `json:"unreliable"`
}

func (i seedIface) graphable() bool { return i.HC && !i.Excluded && !i.Unreliable }

func seedDemoCmd(ctx context.Context, args []string) error {
	cfg := seedConfig{hc: &http.Client{Timeout: 3 * time.Minute}}
	fs := devFlagSet("seed-demo")
	fs.StringVar(&cfg.api, "api", "http://127.0.0.1:8443", "base URL of the running nmon (started with --insecure-http)")
	fs.StringVar(&cfg.hotURL, "hot-url", "", "base URL of the hot VictoriaMetrics, for the back-fill (required)")
	fs.IntVar(&cfg.days, "days", 31, "days of history to back-fill")
	fs.DurationVar(&cfg.step, "step", 60*time.Second, "back-fill sample spacing")
	fs.StringVar(&cfg.countersRec, "counters-rec", "testdata/snmprec/counters.snmprec", "the counters fixture recording")
	fs.StringVar(&cfg.envOut, "env-out", "", "write NMON_DEVICE=<counters slug> and NMON_DEVICE_48=<48-port slug> to this file when ready")
	fs.StringVar(&cfg.profilesDir, "profiles-dir", "", "operator profiles directory (shipped profiles are always loaded)")
	fs.IntVar(&cfg.workers, "workers", 6, "parallel back-fill workers")
	if err := fs.Parse(args); err != nil {
		return flagErr(err)
	}
	if cfg.hotURL == "" {
		return errors.New("--hot-url is required")
	}
	cfg.api, cfg.hotURL = strings.TrimRight(cfg.api, "/"), strings.TrimRight(cfg.hotURL, "/")
	return cfg.run(ctx)
}

func (c *seedConfig) run(ctx context.Context) error {
	set, err := profile.Load(c.profilesDir)
	if err != nil {
		return err
	}
	fleet, err := buildFleet(set, c.countersRec)
	if err != nil {
		return err
	}
	var pool simagent.Pool
	defer func() { _ = pool.Close() }() // process is exiting
	added, err := c.addAll(ctx, &pool, fleet)
	if err != nil {
		return err
	}
	if err := c.backfillAll(ctx, added); err != nil {
		return err
	}
	if err := c.writeEnv(added); err != nil {
		return err
	}
	fmt.Println("seed-demo ready: agents keep answering until interrupted")
	<-ctx.Done()
	return nil
}

// addAll starts one agent per device on [::1] with its own port and adds the
// device through the API with that port.
func (c *seedConfig) addAll(ctx context.Context, pool *simagent.Pool, fleet []demoDevice) ([]seeded, error) {
	var out []seeded
	for i, d := range fleet {
		cfg := simagent.Config{Addr: "[::1]:0", Recording: d.rec, Community: d.community}
		if err := pool.Start(ctx, []simagent.Config{cfg}); err != nil {
			return nil, fmt.Errorf("%s: %w", d.name, err)
		}
		agent := pool.Agents()[i]
		s, err := c.addDevice(ctx, d, "::1", int(agent.Addr().Port()))
		s.agent = agent
		if err != nil {
			return nil, fmt.Errorf("add %s: %w", d.name, err)
		}
		fmt.Printf("added %-12s %-10s %s (%d interfaces)\n", d.name, d.kind, s.slug, len(s.ifaces))
		out = append(out, s)
	}
	return out, nil
}

func (c *seedConfig) addDevice(ctx context.Context, d demoDevice, ip string, port int) (seeded, error) {
	version := d.version
	if version == "" {
		version = "v2c"
	}
	body := map[string]any{
		"name": d.name, "addresses": []string{ip}, "site": d.site, "role": d.role,
		"snmp": map[string]any{"version": version, "community": d.community, "port": port},
	}
	var created struct {
		Slug string `json:"slug"`
	}
	if err := httpJSON(ctx, c.hc, http.MethodPost, c.api+"/api/v1/devices", body, &created); err != nil {
		return seeded{}, err
	}
	s := seeded{demoDevice: d, slug: created.Slug, ip: ip}
	var got struct {
		Interfaces []seedIface `json:"interfaces"`
	}
	if err := httpJSON(ctx, c.hc, http.MethodGet, c.api+"/api/v1/devices/"+url.PathEscape(created.Slug), nil, &got); err != nil {
		return seeded{}, err
	}
	if len(got.Interfaces) == 0 {
		return seeded{}, errors.New("discovery found no interfaces")
	}
	s.ifaces = got.Interfaces
	return s, nil
}

// backfillAll waits for each device's first live samples, then writes history
// for every graphable interface in parallel.
func (c *seedConfig) backfillAll(ctx context.Context, devs []seeded) error {
	type job struct{ dev, iface string }
	var jobs []job
	probe := &backfillConfig{hotURL: c.hotURL, hc: c.hc}
	for _, d := range devs {
		for _, i := range d.ifaces {
			if d.hc && i.graphable() {
				jobs = append(jobs, job{d.slug, i.ID})
			}
		}
		if err := c.waitLive(ctx, probe, d); err != nil {
			return err
		}
		if d.kind == "down" {
			_ = d.agent.Close() // goes silent now, so it reads as down by the time the screenshots are taken
		}
	}
	ch := make(chan job)
	errs := make(chan error, len(jobs))
	var wg sync.WaitGroup
	for w := 0; w < max(1, c.workers); w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for j := range ch {
				cfg := &backfillConfig{hotURL: c.hotURL, device: j.dev, days: c.days, step: c.step, pattern: patternDiurnal,
					seed: 1, hc: c.hc, logf: func(string, ...any) {}}
				errs <- cfg.backfillIface(ctx, j.iface)
			}
		}()
	}
	for _, j := range jobs {
		ch <- j
	}
	close(ch)
	wg.Wait()
	close(errs)
	for err := range errs {
		if err != nil {
			return err
		}
	}
	fmt.Printf("back-filled %d interfaces, %d days at %s\n", len(jobs), c.days, c.step)
	return nil
}

// waitLive blocks until the hot store holds two live samples of the device's
// first graphable interface, so the back-fill can end where they begin.
func (c *seedConfig) waitLive(ctx context.Context, probe *backfillConfig, d seeded) error {
	if !d.hc {
		return nil
	}
	var first string
	for _, i := range d.ifaces {
		if i.graphable() {
			first = i.ID
			break
		}
	}
	if first == "" {
		return nil
	}
	deadline := time.Now().Add(3 * time.Minute)
	for time.Now().Before(deadline) {
		live, err := probe.probe(ctx, "ifHCInOctets", first)
		if err == nil && live.ok && live.rate > 0 {
			return nil
		}
		if !sleepCtx(ctx, 2*time.Second) {
			return ctx.Err()
		}
	}
	return fmt.Errorf("%s: no live samples in the hot store after 3 minutes (is nmon polling? --poll-interval)", d.slug)
}

func (c *seedConfig) writeEnv(devs []seeded) error {
	var b bytes.Buffer
	for _, d := range devs {
		switch d.kind {
		case "counters":
			fmt.Fprintf(&b, "NMON_DEVICE=%s\n", d.slug)
		case "big":
			fmt.Fprintf(&b, "NMON_DEVICE_48=%s\n", d.slug)
		case "v1":
			fmt.Fprintf(&b, "NMON_DEVICE_32BIT=%s\n", d.slug)
		case "down":
			fmt.Fprintf(&b, "NMON_DEVICE_DOWN=%s\n", d.slug)
		}
	}
	if c.envOut == "" {
		return nil
	}
	return os.WriteFile(c.envOut, b.Bytes(), 0o600)
}