nmon/internal/alert/rules.go

// Package alert evaluates metric alert rules against the hot store every 30 s
// and keeps a persisted per-series state machine (pending, firing, resolved,
// stale, flapping). Reachability alerts are not here; they never need the
// metric store.
package alert

import (
	"bytes"
	"errors"
	"fmt"
	"slices"
	"strings"
	"text/template"
	"time"

	"github.com/v10networks/nmon/internal/alertpolicy"
)

// Severities.
const (
	SevCritical = "critical"
	SevWarning  = "warning"
	SevInfo     = "info"
)

// Comparison operators.
const (
	OpGreater      = ">"
	OpLess         = "<"
	OpGreaterEqual = ">="
	OpLessEqual    = "<="
	OpEqual        = "=="
	OpNotEqual     = "!="
)

// validOp reports whether op is one of the six comparisons.
func validOp(op string) bool {
	switch op {
	case OpGreater, OpLess, OpGreaterEqual, OpLessEqual, OpEqual, OpNotEqual:
		return true
	}
	return false
}

// Rule is one metric alert rule. Expr is an instant MetricsQL query that
// returns one series per entity whose value is the measured quantity; the
// evaluator compares it with the threshold (so thresholds can differ per group
// or per device without rewriting the query). A series that is absent from a
// result is "no data", never "below".
type Rule struct {
	Name     string `yaml:"name" json:"name"`
	Severity string `yaml:"severity" json:"severity"`
	Expr     string `yaml:"expr" json:"expr"`
	Op       string `yaml:"op" json:"op"` // > >= < <= == !=
	// Threshold is the default; see Overrides and Evaluator Options.ThresholdFor.
	Threshold float64       `yaml:"threshold" json:"threshold"`
	For       time.Duration `yaml:"for" json:"for"`
	// Labels names the series labels that identify the entity, any of device,
	// iface, component. A series missing one of them is skipped.
	Labels []string `yaml:"labels" json:"labels"`
	// Summary and Description are text/template strings over TemplateData.
	Summary     string `yaml:"summary" json:"summary"`
	Description string `yaml:"description" json:"description"`
	// Overrides are per-group thresholds; the most specific group wins.
	Overrides []Override `yaml:"overrides" json:"overrides"`
	// SkipUnreliable drops series whose interface the Unreliable filter flags
	// (32-bit counters on fast links wrap too quickly to trust).
	SkipUnreliable bool `yaml:"skip_unreliable" json:"skip_unreliable"`
	// OptIn is a no-op kept so old configs still parse. Alert policies (the alert_down matcher) replaced it:
	// a rule fires only where a policy binds it.
	OptIn    bool `yaml:"opt_in" json:"opt_in"`
	Disabled bool `yaml:"disabled" json:"disabled"`
	// Source, when set, supplies the readings instead of Expr (inventory rules, evaluated in Go). Expr is then empty.
	Source Source `yaml:"-" json:"-"`
}

// Override replaces a rule's threshold for devices in a group.
type Override struct {
	Group     string  `yaml:"group" json:"group"`
	Threshold float64 `yaml:"threshold" json:"threshold"`
}

// TemplateData is what Summary and Description templates see.
type TemplateData struct {
	Rule, Severity, Device, Iface, Component string
	Value, Threshold                         float64
	Op                                       string
	For                                      time.Duration
}

// reachRule reports the rule names that other components own in the alerts table: the
// reachability evaluator (internal/reach) and the subsystem alarms (SystemRule). This evaluator
// must neither close nor deliver those rows, and no configured rule may take these names.
func reachRule(name string) bool {
	return name == "device_down" || name == "device_warning" || SystemRule(name)
}

var entityLabels = map[string]bool{"device": true, "iface": true, "component": true}

// Validate checks a rule is usable: names, operator, label set, parseable
// templates and a structurally sound expression.
func (r Rule) Validate() error {
	var errs []error
	add := func(f string, a ...any) {
		errs = append(errs, fmt.Errorf("rule %q: "+f, append([]any{r.Name}, a...)...))
	}
	if strings.TrimSpace(r.Name) == "" || strings.ContainsAny(r.Name, "| \t\n") {
		add("name must be non-empty without spaces or '|'")
	}
	switch r.Severity {
	case SevCritical, SevWarning, SevInfo:
	default:
		add("severity %q is not critical, warning or info", r.Severity)
	}
	if !validOp(r.Op) {
		add("op %q is not one of > >= < <= == !=", r.Op)
	}
	if r.For < 0 {
		add("for is negative")
	}
	if reachRule(r.Name) {
		add("name is reserved for reachability alerts")
	}
	r.validateLabels(add)
	if r.Source == nil {
		if err := checkExprShape(r.Expr); err != nil {
			add("expr: %v", err)
		}
	} else if r.Expr != "" {
		add("a rule with a source takes no expr")
	}
	r.validateOverridesAndTemplates(add)
	return errors.Join(errs...)
}

// validateLabels checks the rule's label set names only entity labels.
func (r Rule) validateLabels(add func(string, ...any)) {
	if len(r.Labels) == 0 {
		add("labels is empty")
	}
	for _, l := range r.Labels {
		if !entityLabels[l] {
			add("label %q is not device, iface or component", l)
		}
	}
}

// validateOverridesAndTemplates checks overrides name a group and the
// summary and description templates parse.
func (r Rule) validateOverridesAndTemplates(add func(string, ...any)) {
	for _, o := range r.Overrides {
		if o.Group == "" {
			add("override without a group")
		}
	}
	for n, t := range map[string]string{"summary": r.Summary, "description": r.Description} {
		if _, err := template.New(n).Option("missingkey=error").Parse(t); err != nil {
			add("%s template: %v", n, err)
		}
	}
}

// checkExprShape is a cheap structural check (non-empty, balanced brackets and
// quotes). The real parser is the store; the integration test runs every
// default rule against it.
func checkExprShape(s string) error {
	if strings.TrimSpace(s) == "" {
		return errors.New("empty")
	}
	var stack []byte
	var quote byte
	for i := 0; i < len(s); i++ {
		c := s[i]
		if quote != 0 {
			switch {
			case c == '\\' && quote != '`':
				i++
			case c == quote:
				quote = 0
			}
			continue
		}
		switch c {
		case '"', '\'', '`':
			quote = c
		case '(', '[', '{':
			stack = append(stack, c)
		case ')', ']', '}':
			want := map[byte]byte{')': '(', ']': '[', '}': '{'}[c]
			if len(stack) == 0 || stack[len(stack)-1] != want {
				return fmt.Errorf("unbalanced %q at offset %d", c, i)
			}
			stack = stack[:len(stack)-1]
		}
	}
	if quote != 0 {
		return errors.New("unterminated string")
	}
	if len(stack) != 0 {
		return fmt.Errorf("unclosed %q", stack[len(stack)-1])
	}
	return nil
}

type compiledRule struct {
	Rule
	summary, desc *template.Template
}

func compile(r Rule) (*compiledRule, error) {
	if err := r.Validate(); err != nil {
		return nil, err
	}
	c := &compiledRule{Rule: r}
	var err error
	if c.summary, err = template.New("summary").Option("missingkey=error").Parse(r.Summary); err != nil {
		return nil, err
	}
	if c.desc, err = template.New("description").Option("missingkey=error").Parse(r.Description); err != nil {
		return nil, err
	}
	return c, nil
}

func (c *compiledRule) render(t *template.Template, d TemplateData) string {
	var b bytes.Buffer
	if err := t.Execute(&b, d); err != nil {
		return t.Name() + " template error: " + err.Error()
	}
	return b.String()
}

// breach reports whether v is on the alerting side of threshold.
func (r Rule) breach(v, threshold float64) bool {
	switch r.Op {
	case OpLess:
		return v < threshold
	case OpLessEqual:
		return v <= threshold
	case OpGreaterEqual:
		return v >= threshold
	case OpEqual:
		return v == threshold
	case OpNotEqual:
		return v != threshold
	}
	return v > threshold
}

// GroupRef is a group a device belongs to. Size is the member count; a smaller
// group is more specific.
type GroupRef struct {
	Name string
	Size int
}

// GroupLookup answers group membership for threshold overrides.
type GroupLookup interface {
	GroupsOf(device string) []GroupRef
}

// resolveThreshold picks the threshold for a device: the override of the most
// specific (smallest) group the device is in, ties going to the earlier
// override in the rule; else thresholdFor (per-profile) if it has one; else the
// rule default.
func (r Rule) resolveThreshold(device string, g GroupLookup, thresholdFor func(device, rule string) (float64, bool)) float64 {
	if g != nil && len(r.Overrides) > 0 {
		size := map[string]int{}
		for _, gr := range g.GroupsOf(device) {
			size[gr.Name] = gr.Size
		}
		best, bestSize, found := 0.0, 0, false
		for _, o := range r.Overrides {
			sz, ok := size[o.Group]
			if ok && (!found || sz < bestSize) {
				best, bestSize, found = o.Threshold, sz, true
			}
		}
		if found {
			return best
		}
	}
	if thresholdFor != nil {
		if v, ok := thresholdFor(device, r.Name); ok {
			return v
		}
	}
	return r.Threshold
}

// bound returns the rule as one entity's policy binding sees it (a copy with the policy's severity and for)
// and the threshold to compare with: the policy's, else the rule's group override, profile or default.
func (c *compiledRule) bound(b alertpolicy.Binding, device string, g GroupLookup,
	thresholdFor func(device, rule string) (float64, bool)) (*compiledRule, float64) {
	rr := *c
	if b.Severity != "" {
		rr.Severity = b.Severity
	}
	if b.For != nil {
		rr.For = *b.For
	}
	if b.Threshold != nil {
		return &rr, *b.Threshold
	}
	return &rr, c.resolveThreshold(device, g, thresholdFor)
}

// Level says what the rule's entity is: an interface, a component of a device, or a device. Policies use it to
// refuse matchers that cannot apply.
func (r Rule) Level() string {
	switch {
	case slices.Contains(r.Labels, "iface"):
		return alertpolicy.LevelInterface
	case slices.Contains(r.Labels, "component"):
		return alertpolicy.LevelComponent
	}
	return alertpolicy.LevelDevice
}

// patchRule overlays the non-zero tunables of c onto d.
func patchRule(d, c Rule) Rule {
	if c.Severity != "" {
		d.Severity = c.Severity
	}
	if c.Op != "" {
		d.Op = c.Op
	}
	if c.Threshold != 0 {
		d.Threshold = c.Threshold
	}
	if c.For != 0 {
		d.For = c.For
	}
	if c.Overrides != nil {
		d.Overrides = c.Overrides
	}
	d.Disabled = c.Disabled
	return d
}

// MergeRules applies configured rules over defaults. A configured rule with
// the name of a default and no Expr patches that default's severity, op,
// threshold, for, overrides and disabled flag (zero values keep the default;
// set Disabled to switch a rule off). A rule with an Expr replaces the default
// wholesale; an unknown name needs a full definition. Order: defaults first,
// then new rules in config order.
func MergeRules(defaults, configured []Rule) ([]Rule, error) {
	out := append([]Rule(nil), defaults...)
	idx := map[string]int{}
	for i, r := range out {
		idx[r.Name] = i
	}
	for _, c := range configured {
		i, known := idx[c.Name]
		switch {
		case known && c.Expr == "":
			out[i] = patchRule(out[i], c)
		case known:
			out[i] = c
		default:
			idx[c.Name] = len(out)
			out = append(out, c)
		}
	}
	for _, r := range out {
		if err := r.Validate(); err != nil {
			return nil, err
		}
	}
	return out, nil
}

const (
	ramRE    = `(?i).*(physical memory|(^|[^a-z])ram([^a-z]|$)).*`
	notDiskE = `(?i).*(memory|swap|buffers|cached|(^|[^a-z])ram([^a-z]|$)).*`
)

// DefaultRules returns the shipped rules. Notes on the less obvious queries:
//
//   - Every gauge is wrapped in last_over_time over a window of at least three
//     poll intervals, so a 5 min poll does not fall out of the store's default
//     5 min lookback and read as "no data".
//   - interface_errors: error ratio in percent. Packets are rate of
//     ifHCIn/OutUcastPkts when the device exports them, else
//     (ifHCInOctets+ifHCOutOctets)/1500 (the shipped profiles carry no packet
//     counters, so the approximation is the usual path and understates packets
//     for small-packet traffic, which only makes the alert earlier). The
//     denominator is floored at 10 pkt/s so an idle port with a single error
//     does not read as 100% (and a zero denominator never yields NaN).
//   - memory_high: UCD memAvailReal/memTotalReal when present, else the
//     hrStorage rows whose component (hrStorageDescr) matches /physical memory|ram/i.
//     hrStorageType is not a label, so the description is all there is. UCD
//     "available" excludes cache, so it reads high on Linux; that is the
//     conventional reading.
//   - disk_full: every hrStorage row that is not memory, swap, buffers or cache.
//     The value is max(used%, 100 if predict_linear over 6 h reaches the size
//     within 24 h; tagged kind=predicted so union keeps both), so one threshold covers both conditions; a threshold of 100
//     or more disables the prediction half in effect.
//   - printer_supply_low: negative levels (-1 other, -2 unknown, -3 "some
//     remaining") are filtered out; the series then goes stale rather than low.
//   - printer_supply_empty: a supply level of exactly 0 (critical); the negative
//     sentinels are not 0, so they never fire it.
//   - temp_high: the temperature metrics of the shipped profiles, each tagged
//     with its metric name as component where the profile has no component
//     label. entPhySensorValue is excluded, since it mixes units. Per-profile
//     thresholds arrive through Options.ThresholdFor; the default is 70.
func DefaultRules() []Rule {
	lab := func(l ...string) []string { return l }
	return append(hostRules(lab), deviceRules(lab)...)
}

// hostRules are the interface, CPU, memory and disk rules of DefaultRules.
func hostRules(lab func(...string) []string) []Rule {
	return []Rule{
		{
			Name: "interface_down", Severity: SevWarning, Op: OpGreater, Threshold: 0, For: 2 * time.Minute,
			Labels:      lab("device", "iface"),
			Expr:        `(ifOperStatus == bool 2) * on(device, iface) (ifAdminStatus == bool 1)`,
			Summary:     `{{.Device}} interface {{.Iface}} is down`,
			Description: `Interface {{.Iface}} on {{.Device}} is operationally down while administratively up (for {{.For}}).`,
		},
		{
			Name: "interface_errors", Severity: SevWarning, Op: OpGreater, Threshold: 0.1, For: 0,
			Labels: lab("device", "iface"), SkipUnreliable: true,
			Expr: `100 * (rate(ifInErrors[10m]) + rate(ifOutErrors[10m])) / clamp_min(` +
				`(rate(ifHCInUcastPkts[10m]) + rate(ifHCOutUcastPkts[10m])) or ` +
				`((rate(ifHCInOctets[10m]) + rate(ifHCOutOctets[10m])) / 1500), 10)`,
			Summary: `{{.Device}} interface {{.Iface}} error ratio {{printf "%.2f" .Value}}%`,
			Description: `Errors are {{printf "%.2f" .Value}}% of packets over 10 minutes on {{.Iface}} of {{.Device}} ` +
				`(threshold {{printf "%.2f" .Threshold}}%).`,
		},
		{
			Name: "cpu_high", Severity: SevWarning, Op: OpGreater, Threshold: 90, For: 10 * time.Minute,
			Labels:      lab("device"),
			Expr:        `avg by (device) (last_over_time(hrProcessorLoad[15m]))`,
			Summary:     `{{.Device}} CPU at {{printf "%.0f" .Value}}%`,
			Description: `Average processor load on {{.Device}} is {{printf "%.1f" .Value}}% (threshold {{printf "%.0f" .Threshold}}%).`,
		},
		{
			Name: "memory_high", Severity: SevWarning, Op: OpGreater, Threshold: 90, For: 10 * time.Minute,
			Labels: lab("device"),
			Expr: `(100 * (1 - last_over_time(memAvailReal[15m]) / (last_over_time(memTotalReal[15m]) > 0))) or on(device) ` +
				`(max by (device) (100 * last_over_time(hrStorageUsed{component=~"` + ramRE + `"}[15m]) / ` +
				`(last_over_time(hrStorageSize{component=~"` + ramRE + `"}[15m]) > 0)))`,
			Summary:     `{{.Device}} memory at {{printf "%.0f" .Value}}%`,
			Description: `Memory use on {{.Device}} is {{printf "%.1f" .Value}}% (threshold {{printf "%.0f" .Threshold}}%).`,
		},
		{
			Name: "disk_full", Severity: SevCritical, Op: OpGreater, Threshold: 90, For: 0,
			Labels: lab("device", "component"),
			Expr: `max by (device, component) (union(` +
				`100 * last_over_time(hrStorageUsed{component!~"` + notDiskE + `"}[15m]) / ` +
				`(last_over_time(hrStorageSize{component!~"` + notDiskE + `"}[15m]) > 0), ` +
				`label_set(100 * (predict_linear(hrStorageUsed{component!~"` + notDiskE + `"}[6h], 86400) > bool ` +
				`last_over_time(hrStorageSize{component!~"` + notDiskE + `"}[15m])), "kind", "predicted")))`,
			Summary: `{{.Device}} {{.Component}} is {{printf "%.0f" .Value}}% full or filling`,
			Description: `Storage {{.Component}} on {{.Device}} is above {{printf "%.0f" .Threshold}}% used or ` +
				`predicted to fill within 24 hours (value {{printf "%.1f" .Value}}).`,
		},
	}
}

// deviceRules are the UPS, printer and temperature rules of DefaultRules.
func deviceRules(lab func(...string) []string) []Rule {
	return []Rule{
		{
			Name: "ups_on_battery", Severity: SevCritical, Op: OpGreater, Threshold: 0, For: 0,
			Labels:      lab("device"),
			Expr:        `max by (device) (last_over_time(upsOutputSource[15m]) == bool 5)`,
			Summary:     `{{.Device}} UPS is on battery`,
			Description: `The UPS {{.Device}} reports its output source as battery.`,
		},
		{
			Name: "ups_runtime_low", Severity: SevCritical, Op: OpLess, Threshold: 10, For: time.Minute,
			Labels: lab("device"),
			Expr: `min by (device) (last_over_time(upsEstimatedMinutesRemaining[15m])) or on(device) ` +
				`min by (device) (last_over_time(upsAdvBatteryRunTimeRemaining[15m]) / 60)`,
			Summary:     `{{.Device}} UPS runtime {{printf "%.0f" .Value}} min`,
			Description: `The UPS {{.Device}} estimates {{printf "%.1f" .Value}} minutes of runtime (threshold {{printf "%.0f" .Threshold}}).`,
		},
		{
			Name: "printer_supply_low", Severity: SevWarning, Op: OpLess, Threshold: 10, For: 0,
			Labels: lab("device", "component"),
			Expr: `100 * (last_over_time(prtMarkerSuppliesLevel[1h]) >= 0) / ` +
				`(last_over_time(prtMarkerSuppliesMaxCapacity[1h]) > 0)`,
			Summary:     `{{.Device}} {{.Component}} at {{printf "%.0f" .Value}}%`,
			Description: `Supply {{.Component}} on {{.Device}} is at {{printf "%.0f" .Value}}% (threshold {{printf "%.0f" .Threshold}}%).`,
		},
		{
			Name: "printer_supply_empty", Severity: SevCritical, Op: OpGreater, Threshold: 0, For: 0,
			Labels:      lab("device", "component"),
			Expr:        `last_over_time(prtMarkerSuppliesLevel[1h]) == bool 0`,
			Summary:     `{{.Device}} {{.Component}} is empty`,
			Description: `Supply {{.Component}} on {{.Device}} reports a level of 0 (empty).`,
		},
		{
			Name: "temp_high", Severity: SevWarning, Op: OpGreater, Threshold: 70, For: 5 * time.Minute,
			Labels: lab("device", "component"),
			Expr: `union(` +
				`label_set(last_over_time(mtxrHlTemperature[15m]), "component", "mtxrHlTemperature"), ` +
				`label_set(last_over_time(mtxrHlProcessorTemperature[15m]), "component", "mtxrHlProcessorTemperature"), ` +
				`label_set(last_over_time(upsAdvBatteryTemperature[15m]), "component", "upsAdvBatteryTemperature"), ` +
				`last_over_time(jnxOperatingTemp[15m]))`,
			Summary:     `{{.Device}} {{.Component}} at {{printf "%.0f" .Value}} C`,
			Description: `Temperature {{.Component}} on {{.Device}} is {{printf "%.1f" .Value}} C (threshold {{printf "%.0f" .Threshold}}).`,
		},
		{
			// state 6 is established in every BGP MIB the profiles read (BGP4-MIB, CISCO-BGP4, Juniper BGP4-V2, Arista
			// BGP4V2). The notification adds the peer's AS and description from the component attributes.
			Name: "bgp_peer_down", Severity: SevCritical, Op: OpGreater, Threshold: 0, For: 2 * time.Minute,
			Labels:      lab("device", "component"),
			Expr:        `last_over_time(bgp_peer_state[15m]) != bool 6`,
			Summary:     `{{.Device}} BGP peer {{.Component}} is down`,
			Description: `The BGP session to {{.Component}} on {{.Device}} has not been established for {{.For}}.`,
		},
	}
}