nmon/internal/api/reach.go

package api

import (
	"context"
	"time"

	"github.com/v10networks/nmon/internal/clock"
	"github.com/v10networks/nmon/internal/store"
)

// Reachability values shown in lists and on device pages.
const (
	ReachUp          = "up"
	ReachDown        = "down"
	ReachWarning     = "warning"
	ReachUnknown     = "unknown"
	ReachMaintenance = "maintenance"
)

// ReachabilitySource decides each device's reachability. The result is keyed by
// device slug and holds one of the Reach* values. M4 replaces the default with
// the alerter's state (dependencies, maintenance windows, suppression).
type ReachabilitySource interface {
	Reachability(ctx context.Context, devs []store.Device) map[string]string
}

// persistedReachability derives reachability from what is stored right now:
// the ICMP and SNMP up series in the hot store, else the device's last_seen
// age against the poll interval. It knows nothing of dependencies.
type persistedReachability struct {
	q        Querier
	clk      clock.Clock
	cadence  *store.PollCadence
	interval time.Duration
}

// upLookback is how stale an up series may be and still count.
// It must outlast two default SNMP polls, or nmon_snmp_up would drop out between polls.
const upLookback = "11m"

// maxScoped is the most devices a query names explicitly; beyond it all devices are fetched.
const maxScoped = 200

func (p *persistedReachability) Reachability(ctx context.Context, devs []store.Device) map[string]string {
	icmp, snmp := p.upSeries(ctx, devs)
	now := p.clk.Now()
	out := make(map[string]string, len(devs))
	for i := range devs {
		d := &devs[i]
		out[d.Slug] = reachabilityOf(d, icmp[d.Slug], snmp[d.Slug], now, d.PollInterval(p.cadence.Or(p.interval)))
	}
	return out
}

// upSeries returns the last nmon_icmp_up and nmon_snmp_up per device slug. A
// store that cannot be queried yields empty maps, and the caller falls back to last_seen.
func (p *persistedReachability) upSeries(ctx context.Context, devs []store.Device) (icmp, snmp map[string]*float64) {
	match := ""
	if len(devs) > 0 && len(devs) <= maxScoped {
		slugs := make([]string, len(devs))
		for i, d := range devs {
			slugs[i] = d.Slug
		}
		match = `device=~` + quote(regexAlt(slugs))
	}
	get := func(metric string) map[string]*float64 {
		body, err := p.q.Query(ctx, "hot", `last_over_time(`+metric+`{`+match+`}[`+upLookback+`])`, p.clk.Now())
		if err != nil {
			return nil
		}
		rows, err := parseVM(body)
		if err != nil {
			return nil
		}
		m := make(map[string]*float64, len(rows))
		for _, r := range rows {
			if r.Value != nil && r.Value.OK {
				v := r.Value.V
				m[r.Metric["device"]] = &v
			}
		}
		return m
	}
	return get("nmon_icmp_up"), get("nmon_snmp_up")
}

// reachabilityOf is the decision for one device. icmp and snmp are the last up
// values (1 or 0) or nil when the store has none.
func reachabilityOf(d *store.Device, icmp, snmp *float64, now time.Time, interval time.Duration) string {
	switch {
	case d.RetiredAt != nil || !d.Enabled:
		return ReachUnknown
	case d.PollState == store.PollMaintenance:
		return ReachMaintenance
	}
	var probes []bool
	for _, v := range []*float64{icmp, snmp} {
		if v != nil {
			probes = append(probes, *v >= 1)
		}
	}
	if len(probes) == 0 {
		return staleness(d, now, interval)
	}
	up := 0
	for _, ok := range probes {
		if ok {
			up++
		}
	}
	switch {
	case up == 0:
		return ReachDown
	case up < len(probes), d.PollState == store.PollDegraded, d.PollState == store.PollBackoff:
		return ReachWarning
	}
	return ReachUp
}

// staleness judges a device with no up series by when it last answered: not
// heard from in three poll intervals is down.
func staleness(d *store.Device, now time.Time, interval time.Duration) string {
	if d.LastSeen == nil {
		return ReachUnknown
	}
	if now.Sub(*d.LastSeen) > 3*interval {
		return ReachDown
	}
	if d.PollState == store.PollDegraded || d.PollState == store.PollBackoff {
		return ReachWarning
	}
	return ReachUp
}

// pollStateOf maps the stored state to the API enum (ok, degraded, backoff);
// maintenance is a reachability, not a poll state.
func pollStateOf(d *store.Device) string {
	switch d.PollState {
	case store.PollDegraded, store.PollBackoff:
		return d.PollState
	}
	return store.PollOK
}