The predefined rules are too constraining: an ISP cannot say "CPU over 80% on servers for 15 minutes" or "uplink without a description", and every new need meant editing config.yaml. Admins now build their own rules; the shipped ones stay as read-only built-in macros with their names unchanged, so v1.1 policies and alert history keep working. Custom rules live in a new alert_rules table (migration 13) and are builder-only. A metric rule names a catalogue metric (every pollable profile metric plus push-ingested names) with a transform (value, rate, percent_of), window, function, aggregate, comparison and threshold; the pure internal/alertrule package compiles it to MetricsQL from fixed templates, so no user text reaches a query. An inventory rule ANDs one to five conditions over whitelisted device or port fields and is evaluated in Go over store rows. The evaluator takes its rule set from a function each cycle, so edits apply live, and both kinds use the existing state machine, policies, suppression and notifications. Comparisons now cover > >= < <= == != everywhere. utilisation is not offered: port speed is not a stored series. config.yaml rules still work but are deprecated. API (additive): rule list gains id/kind/editable/spec/expr, plus get, create, update, delete, catalogue, test (viewer, read-only) and copy of a built-in (cpu_high, ups_on_battery, printer_supply_empty) to a custom rule. Deleting or renaming a rule that a policy names is a 409. The UI gets an Alert rules tab with a builder dialog, a test button and the generated query, and the policy dialog lists custom rules. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T4PdJUPrakM8fHvN8jYv2x
338 lines
10 KiB
Go
338 lines
10 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/v10networks/nmon/internal/alert"
|
|
"github.com/v10networks/nmon/internal/alertpolicy"
|
|
"github.com/v10networks/nmon/internal/alertrule"
|
|
"github.com/v10networks/nmon/internal/api"
|
|
"github.com/v10networks/nmon/internal/profile"
|
|
"github.com/v10networks/nmon/internal/selfmon"
|
|
"github.com/v10networks/nmon/internal/store"
|
|
"github.com/v10networks/nmon/profiles"
|
|
)
|
|
|
|
// ruleQuerier answers instant queries from a table of expr -> device -> value, and the push name query.
|
|
type ruleQuerier struct {
|
|
mu sync.Mutex
|
|
series map[string]map[string]float64
|
|
push []string
|
|
}
|
|
|
|
func (q *ruleQuerier) set(expr, device string, v float64) {
|
|
q.mu.Lock()
|
|
defer q.mu.Unlock()
|
|
if q.series == nil {
|
|
q.series = map[string]map[string]float64{}
|
|
}
|
|
if q.series[expr] == nil {
|
|
q.series[expr] = map[string]float64{}
|
|
}
|
|
q.series[expr][device] = v
|
|
}
|
|
|
|
func (q *ruleQuerier) Query(_ context.Context, _, expr string, _ time.Time) ([]byte, error) {
|
|
q.mu.Lock()
|
|
defer q.mu.Unlock()
|
|
var res []map[string]any
|
|
if expr == pushNamesQuery {
|
|
for _, n := range q.push {
|
|
res = append(res, map[string]any{"metric": map[string]string{"__name__": n}, "value": []any{1, "1"}})
|
|
}
|
|
}
|
|
for dev, v := range q.series[expr] {
|
|
res = append(res, map[string]any{"metric": map[string]string{"device": dev}, "value": []any{1, jsonNum(v)}})
|
|
}
|
|
return json.Marshal(map[string]any{"status": "success", "data": map[string]any{"resultType": "vector", "result": res}})
|
|
}
|
|
|
|
func jsonNum(v float64) string { b, _ := json.Marshal(v); return string(b) }
|
|
|
|
type ruleNotifier struct {
|
|
mu sync.Mutex
|
|
states []string
|
|
}
|
|
|
|
func (n *ruleNotifier) Notify(_ context.Context, t alert.Transition) error {
|
|
n.mu.Lock()
|
|
defer n.mu.Unlock()
|
|
n.states = append(n.states, t.Rule+":"+t.State)
|
|
return nil
|
|
}
|
|
|
|
func (n *ruleNotifier) got() string {
|
|
n.mu.Lock()
|
|
defer n.mu.Unlock()
|
|
return strings.Join(n.states, ",")
|
|
}
|
|
|
|
type ruleEnv struct {
|
|
*policyEnv
|
|
q *ruleQuerier
|
|
n *ruleNotifier
|
|
ev *alert.Evaluator
|
|
set *profile.Set
|
|
}
|
|
|
|
func newRuleEnv(t *testing.T) *ruleEnv {
|
|
t.Helper()
|
|
e := &ruleEnv{policyEnv: newPolicyEnv(t), q: &ruleQuerier{}, n: &ruleNotifier{}}
|
|
var err error
|
|
if e.set, err = profile.LoadFS(profiles.FS); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e.reg = newRuleRegistry(e.st, e.clk, alert.DefaultRules(), time.Minute, e.q, func() *profile.Set { return e.set }, testLogger())
|
|
e.pe = newPolicyEngine(e.st, e.clk, newOptInSet(e.st, e.clk, entityTTL, testLogger()), e.reg, testLogger())
|
|
e.ev, err = alert.New(alert.Options{Store: e.st, Querier: e.q, Notifier: e.n, Clock: e.clk, RuleSet: e.reg.Set,
|
|
Policy: e.pe.Metric, Metrics: selfmon.New(), Logger: testLogger()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return e
|
|
}
|
|
|
|
func (e *ruleEnv) addRule(name, kind string, spec alertrule.RuleSpec) *store.AlertRule {
|
|
e.t.Helper()
|
|
js, err := json.Marshal(spec)
|
|
if err != nil {
|
|
e.t.Fatal(err)
|
|
}
|
|
r := &store.AlertRule{Name: name, Kind: kind, Enabled: true, Severity: "warning", Spec: string(js)}
|
|
if err := e.st.CreateAlertRule(context.Background(), r); err != nil {
|
|
e.t.Fatal(err)
|
|
}
|
|
e.reg.Changed()
|
|
return r
|
|
}
|
|
|
|
func (e *ruleEnv) cycle() {
|
|
e.clk.Advance(30 * time.Second)
|
|
e.ev.EvalOnce(context.Background())
|
|
}
|
|
|
|
func cpuSpec(threshold float64) alertrule.RuleSpec {
|
|
return alertrule.RuleSpec{Metric: &alertrule.MetricSpec{Metric: "hrProcessorLoad", Aggregate: alertrule.AggMax, Op: ">", Threshold: threshold}}
|
|
}
|
|
|
|
func (e *ruleEnv) exprOf(rule string) string {
|
|
e.t.Helper()
|
|
for _, r := range e.reg.Set() {
|
|
if r.Name == rule {
|
|
return r.Expr
|
|
}
|
|
}
|
|
e.t.Fatalf("rule %s not in the rule set", rule)
|
|
return ""
|
|
}
|
|
|
|
func TestCustomMetricRuleBoundByAPolicyFiresResolvesAndEditsApplyLive(t *testing.T) {
|
|
e := newRuleEnv(t)
|
|
row := e.addRule("cpu_custom", alertrule.KindMetric, cpuSpec(80))
|
|
e.policy("cpu everywhere", "cpu_custom", alertpolicy.ActionAlert, 0, alertpolicy.Scope{All: true})
|
|
expr := e.exprOf("cpu_custom")
|
|
if expr != `max by (device) (last_over_time(hrProcessorLoad[10m]))` {
|
|
t.Fatalf("expr = %q", expr)
|
|
}
|
|
e.q.set(expr, e.dev.Slug, 95)
|
|
e.cycle()
|
|
if got := e.n.got(); got != "cpu_custom:firing" {
|
|
t.Fatalf("after breach: %q", got)
|
|
}
|
|
e.q.set(expr, e.dev.Slug, 50)
|
|
e.cycle()
|
|
if got := e.n.got(); got != "cpu_custom:firing,cpu_custom:resolved" {
|
|
t.Fatalf("after recovery: %q", got)
|
|
}
|
|
// An edit (threshold 40) applies on the very next cycle, no restart: 50 now breaches.
|
|
js, _ := json.Marshal(cpuSpec(40))
|
|
row.Spec = string(js)
|
|
if err := e.st.UpdateAlertRule(context.Background(), row); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e.reg.Changed()
|
|
e.cycle()
|
|
if got := e.n.got(); !strings.HasSuffix(got, "cpu_custom:resolved,cpu_custom:firing") {
|
|
t.Fatalf("after edit: %q", got)
|
|
}
|
|
// Disabling retires the open alert silently: no resolved notification.
|
|
row.Enabled = false
|
|
if err := e.st.UpdateAlertRule(context.Background(), row); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
e.reg.Changed()
|
|
before := e.n.got()
|
|
e.cycle()
|
|
if e.n.got() != before {
|
|
t.Fatalf("disable notified: %q", e.n.got())
|
|
}
|
|
open, err := e.st.ListActiveAlertStates(context.Background())
|
|
if err != nil || len(open) != 0 {
|
|
t.Fatalf("open alerts after disable: %v %v", open, err)
|
|
}
|
|
}
|
|
|
|
func TestInventoryRuleFiresForTheMatchingPortOnly(t *testing.T) {
|
|
e := newRuleEnv(t)
|
|
id := func(idx int64, name, alias string, mbps int64) string {
|
|
i := e.iface(idx, name, alias, 6)
|
|
got, err := e.st.GetInterface(context.Background(), i)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got.IfHighSpeed = mbps
|
|
if err := e.st.UpsertInterfaces(context.Background(), e.dev.ID, []store.Interface{*got}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return i
|
|
}
|
|
bare := id(1, "uplink1", "", 10000)
|
|
_ = id(2, "uplink2", "to core", 10000)
|
|
_ = id(3, "access1", "", 1000)
|
|
e.addRule("uplink_no_desc", alertrule.KindInventory, alertrule.RuleSpec{Inventory: &alertrule.InventorySpec{Conditions: []alertrule.InventoryCondition{
|
|
{Field: "if_alias", Op: "empty"}, {Field: "if_speed_mbps", Op: ">=", Value: "10000"}}}})
|
|
e.policy("all ports", "uplink_no_desc", alertpolicy.ActionAlert, 0, alertpolicy.Scope{All: true})
|
|
e.cycle()
|
|
as, _, err := e.st.ListAlerts(context.Background(), store.AlertFilter{}, store.Page{Limit: 100})
|
|
if err != nil || len(as) != 1 || as[0].Iface != bare || as[0].State != store.AlertFiring {
|
|
t.Fatalf("alerts: %+v %v", as, err)
|
|
}
|
|
if info := findInfo(e.reg.Rules(), "uplink_no_desc"); info.Level != "interface" || info.Kind != "inventory" || !info.Editable {
|
|
t.Fatalf("info: %+v", info)
|
|
}
|
|
}
|
|
|
|
func findInfo(infos []api.AlertRuleInfo, name string) api.AlertRuleInfo {
|
|
for _, i := range infos {
|
|
if i.Name == name {
|
|
return i
|
|
}
|
|
}
|
|
return api.AlertRuleInfo{}
|
|
}
|
|
|
|
func TestCustomRuleNamesCannotShadowBuiltinsOrConfig(t *testing.T) {
|
|
e := newRuleEnv(t)
|
|
for _, n := range []string{"cpu_high", "device_down", "device_warning", "system", "interface_errors"} {
|
|
if !e.reg.Reserved(n) {
|
|
t.Errorf("%s is not reserved", n)
|
|
}
|
|
}
|
|
if e.reg.Reserved("my_rule") {
|
|
t.Error("my_rule reserved")
|
|
}
|
|
// A stored row that collides is ignored, so the built-in keeps its meaning.
|
|
e.addRule("cpu_high", alertrule.KindMetric, cpuSpec(1))
|
|
n := 0
|
|
for _, r := range e.reg.Set() {
|
|
if r.Name == "cpu_high" {
|
|
n++
|
|
if r.Threshold != 90 {
|
|
t.Fatalf("built-in overridden: %+v", r)
|
|
}
|
|
}
|
|
}
|
|
if n != 1 {
|
|
t.Fatalf("cpu_high appears %d times", n)
|
|
}
|
|
}
|
|
|
|
func TestCopyBuiltinsToTheBuilder(t *testing.T) {
|
|
e := newRuleEnv(t)
|
|
ctx := context.Background()
|
|
d, why := e.reg.Copy(ctx, "cpu_high")
|
|
if why != "" {
|
|
t.Fatal(why)
|
|
}
|
|
b, err := alertrule.Build(d, e.reg.catalogue(ctx), time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if want := alert.DefaultRules()[2].Expr; b.Expr != want {
|
|
t.Fatalf("copy compiles to %q, built-in is %q", b.Expr, want)
|
|
}
|
|
for _, name := range []string{"ups_on_battery", "printer_supply_empty"} {
|
|
if _, why := e.reg.Copy(ctx, name); why != "" {
|
|
t.Errorf("%s: %s", name, why)
|
|
}
|
|
}
|
|
for _, name := range []string{"interface_down", "interface_errors", "memory_high", "disk_full", "ups_runtime_low", "printer_supply_low",
|
|
"temp_high", "device_down", "system"} {
|
|
if _, why := e.reg.Copy(ctx, name); why == "" {
|
|
t.Errorf("%s was copyable", name)
|
|
}
|
|
}
|
|
if _, why := e.reg.Copy(ctx, "no_such"); why == "" {
|
|
t.Error("unknown rule copyable")
|
|
}
|
|
}
|
|
|
|
func TestCatalogueHasProfileAndPushMetrics(t *testing.T) {
|
|
e := newRuleEnv(t)
|
|
e.q.push = []string{"room_temp", `bad"name`}
|
|
e.reg.cat.invalidate()
|
|
cat := e.reg.catalogue(context.Background())
|
|
for _, n := range []string{"hrProcessorLoad", "ifHCInOctets", "ifOperStatus", "room_temp", "prtMarkerSuppliesLevel"} {
|
|
if _, ok := cat.Lookup(n); !ok {
|
|
t.Errorf("%s missing from the catalogue", n)
|
|
}
|
|
}
|
|
if m, _ := cat.Lookup("room_temp"); m.Source != alertrule.SourcePush {
|
|
t.Errorf("room_temp source = %q", m.Source)
|
|
}
|
|
if _, ok := cat.Lookup(`bad"name`); ok {
|
|
t.Error("a name that is not an identifier entered the catalogue")
|
|
}
|
|
// Port speed is not a stored series, so utilisation is not offered.
|
|
for _, m := range cat.List() {
|
|
for _, tr := range m.Transforms {
|
|
if tr == "utilisation" {
|
|
t.Fatalf("%s offers utilisation", m.Name)
|
|
}
|
|
}
|
|
}
|
|
if _, ok := cat.Lookup("ifHighSpeed"); ok {
|
|
t.Error("ifHighSpeed is polled as a series after all; revisit utilisation")
|
|
}
|
|
}
|
|
|
|
func TestTestRunReportsBreachingEntities(t *testing.T) {
|
|
e := newRuleEnv(t)
|
|
ms := cpuSpec(80)
|
|
d := alertrule.Draft{Name: "draft", Kind: alertrule.KindMetric, Severity: "warning", Spec: ms}
|
|
expr := `max by (device) (last_over_time(hrProcessorLoad[10m]))`
|
|
e.q.set(expr, "a", 95)
|
|
e.q.set(expr, "b", 10)
|
|
res, err := e.reg.Test(context.Background(), d)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res.Expr != expr || res.Entities != 2 || res.Breaching != 1 || res.Samples[0].Device != "a" || !res.Samples[0].Breach {
|
|
t.Fatalf("result: %+v", res)
|
|
}
|
|
d.Spec.Metric.Metric = "nope"
|
|
if _, err := e.reg.Test(context.Background(), d); err == nil {
|
|
t.Fatal("unknown metric tested")
|
|
}
|
|
}
|
|
|
|
// TestTemplateDataMirrorsTheEvaluators keeps alertrule's save-time template check in step with the data the
|
|
// evaluator renders with.
|
|
func TestTemplateDataMirrorsTheEvaluators(t *testing.T) {
|
|
a, b := reflect.TypeOf(alert.TemplateData{}), reflect.TypeOf(alertrule.TemplateData{})
|
|
if a.NumField() != b.NumField() {
|
|
t.Fatalf("%d fields vs %d", a.NumField(), b.NumField())
|
|
}
|
|
for i := 0; i < a.NumField(); i++ {
|
|
if a.Field(i).Name != b.Field(i).Name || a.Field(i).Type != b.Field(i).Type {
|
|
t.Errorf("field %d: %v vs %v", i, a.Field(i), b.Field(i))
|
|
}
|
|
}
|
|
}
|