Files
nmon/internal/alert/query.go
Vibe Coder b529423324 Add custom alert rules built from metrics and inventory fields
The predefined rules are too constraining: an ISP cannot say "CPU over
80% on servers for 15 minutes" or "uplink without a description", and
every new need meant editing config.yaml. Admins now build their own
rules; the shipped ones stay as read-only built-in macros with their
names unchanged, so v1.1 policies and alert history keep working.

Custom rules live in a new alert_rules table (migration 13) and are
builder-only. A metric rule names a catalogue metric (every pollable
profile metric plus push-ingested names) with a transform (value, rate,
percent_of), window, function, aggregate, comparison and threshold; the
pure internal/alertrule package compiles it to MetricsQL from fixed
templates, so no user text reaches a query. An inventory rule ANDs one to
five conditions over whitelisted device or port fields and is evaluated
in Go over store rows. The evaluator takes its rule set from a function
each cycle, so edits apply live, and both kinds use the existing state
machine, policies, suppression and notifications. Comparisons now cover
> >= < <= == != everywhere. utilisation is not offered: port speed is not
a stored series. config.yaml rules still work but are deprecated.

API (additive): rule list gains id/kind/editable/spec/expr, plus get,
create, update, delete, catalogue, test (viewer, read-only) and copy of
a built-in (cpu_high, ups_on_battery, printer_supply_empty) to a custom
rule. Deleting or renaming a rule that a policy names is a 409. The UI
gets an Alert rules tab with a builder dialog, a test button and the
generated query, and the policy dialog lists custom rules.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T4PdJUPrakM8fHvN8jYv2x
2026-10-24 00:17:48 +00:00

156 lines
4.1 KiB
Go

package alert
import (
"context"
"encoding/json"
"fmt"
"math"
"strconv"
"time"
"github.com/v10networks/nmon/internal/alertpolicy"
)
// query runs a rule and returns one reading per active, trusted entity.
func (ev *Evaluator) query(ctx context.Context, r *compiledRule, now time.Time) ([]reading, error) {
var rds []Reading
var err error
if r.Source != nil {
rds, err = r.Source.Read(ctx, now)
} else {
rds, err = readMetric(ctx, ev.o.Querier, ev.o.StoreName, r, now)
}
if err != nil {
return nil, err
}
best := map[string]reading{}
var order []string
for _, s := range rds {
ent, bind, ok := ev.entityOfReading(r, s)
if !ok {
continue
}
k := ent.Key()
cur, seen := best[k]
if !seen {
order = append(order, k)
}
// Duplicate series for one entity: keep the one nearer to alerting.
if !seen || r.nearer(s.Value, cur.value) {
best[k] = reading{ent, s.Value, bind}
}
}
out := make([]reading, 0, len(order))
for _, k := range order {
out = append(out, best[k])
}
return out, nil
}
// readMetric runs the rule's expression and maps each series to the entity its labels name. Series lacking a
// label the rule needs are dropped.
func readMetric(ctx context.Context, q Querier, storeName string, r *compiledRule, now time.Time) ([]Reading, error) {
qctx, cancel := context.WithTimeout(ctx, queryTimeout)
defer cancel()
raw, err := q.Query(qctx, storeName, r.Expr, now)
if err != nil {
return nil, err
}
series, err := parseVector(raw)
if err != nil {
return nil, err
}
out := make([]Reading, 0, len(series))
for _, s := range series {
rd := Reading{Value: s.value}
complete := true
for _, l := range r.Labels {
v := s.labels[l]
if v == "" {
complete = false
break
}
switch l {
case "device":
rd.Device = v
case "iface":
rd.Iface = v
case "component":
rd.Component = v
}
}
if complete {
out = append(out, rd)
}
}
return out, nil
}
// nearer reports whether a is closer to breaching than b.
func (r *compiledRule) nearer(a, b float64) bool {
if r.Op == OpLess || r.Op == OpLessEqual {
return a < b
}
return a > b
}
// entityOfReading maps a reading to its entity. It reports false for readings on unreliable interfaces, for
// inactive entities and for entities no alert policy binds.
func (ev *Evaluator) entityOfReading(r *compiledRule, s Reading) (Entity, alertpolicy.Binding, bool) {
ent := Entity{Rule: r.Name, Device: s.Device, Iface: s.Iface, Component: s.Component}
if r.SkipUnreliable && ent.Iface != "" && ev.o.Unreliable != nil && ev.o.Unreliable(ent.Iface) {
return Entity{}, alertpolicy.Binding{}, false
}
if ev.o.Active != nil && !ev.o.Active(ent) {
return Entity{}, alertpolicy.Binding{}, false
}
b, ok := ev.o.Policy(ent)
return ent, b, ok
}
type sample struct {
labels map[string]string
value float64
}
// parseVector reads an instant-query response. NaN samples are dropped (no data).
func parseVector(raw []byte) ([]sample, error) {
var resp struct {
Status string `json:"status"`
ErrorType string `json:"errorType"`
Error string `json:"error"`
Data struct {
ResultType string `json:"resultType"`
Result []struct {
Metric map[string]string `json:"metric"`
Value [2]json.RawMessage `json:"value"`
} `json:"result"`
} `json:"data"`
}
if err := json.Unmarshal(raw, &resp); err != nil {
return nil, fmt.Errorf("alert: bad query response: %w", err)
}
if resp.Status != "success" {
return nil, fmt.Errorf("alert: query status %q: %s %s", resp.Status, resp.ErrorType, resp.Error)
}
if resp.Data.ResultType != "vector" {
return nil, fmt.Errorf("alert: query returned %q, want vector", resp.Data.ResultType)
}
var out []sample
for _, r := range resp.Data.Result {
var str string
if err := json.Unmarshal(r.Value[1], &str); err != nil {
return nil, fmt.Errorf("alert: bad sample value %s", r.Value[1])
}
v, err := strconv.ParseFloat(str, 64)
if err != nil {
return nil, fmt.Errorf("alert: bad sample value %q", str)
}
if math.IsNaN(v) {
continue
}
out = append(out, sample{r.Metric, v})
}
return out, nil
}