nmon/internal/api/alert_rules_test.go

package api_test

import (
	"context"
	"encoding/json"
	"net/http"
	"strings"
	"testing"
	"time"

	"github.com/v10networks/nmon/internal/alertrule"
	"github.com/v10networks/nmon/internal/api"
	"github.com/v10networks/nmon/internal/store"
)

// fakeRuleHost stands in for the registry: one built-in plus the custom rules in the store.
type fakeRuleHost struct {
	st      store.Store
	changes int
	cat     alertrule.Catalogue
}

func newFakeRuleHost() *fakeRuleHost {
	return &fakeRuleHost{cat: alertrule.NewCatalogue([]alertrule.MetricInfo{
		{Name: "hrProcessorLoad", Type: "gauge", Level: alertrule.LevelComponent, IntervalSeconds: 60},
		{Name: "memTotalReal", Type: "gauge", Level: alertrule.LevelDevice, IntervalSeconds: 60},
	})}
}

func (f *fakeRuleHost) Rules() []api.AlertRuleInfo {
	thr := 90.0
	out := []api.AlertRuleInfo{{ID: "cpu_high", Name: "cpu_high", Kind: "builtin", Enabled: true, Level: "device", Op: ">", Threshold: &thr,
		Severity: "warning", Copyable: true}}
	rows, _ := f.st.ListAlertRules(context.Background())
	for _, r := range rows {
		var sp alertrule.RuleSpec
		_ = json.Unmarshal([]byte(r.Spec), &sp)
		level := "device"
		if r.Kind == "inventory" && strings.Contains(r.Spec, "if_") {
			level = "interface"
		}
		out = append(out, api.AlertRuleInfo{ID: r.ID, Name: r.Name, Kind: r.Kind, Editable: true, Enabled: r.Enabled, Level: level,
			Severity: r.Severity, Spec: &sp})
	}
	return out
}

func (f *fakeRuleHost) Catalogue(context.Context) api.RuleCatalogue {
	return api.RuleCatalogue{Metrics: f.cat.List(), Fields: alertrule.Fields(), Ops: alertrule.Ops}
}

func (f *fakeRuleHost) Build(_ context.Context, d alertrule.Draft) (api.RuleBuilt, error) {
	b, err := alertrule.Build(d, f.cat, time.Minute)
	if err != nil {
		return api.RuleBuilt{}, &api.RuleInvalid{Err: err}
	}
	return api.RuleBuilt{Expr: b.Expr, Level: b.Level}, nil
}

func (f *fakeRuleHost) Test(ctx context.Context, d alertrule.Draft) (api.RuleTestResult, error) {
	b, err := f.Build(ctx, d)
	if err != nil {
		return api.RuleTestResult{}, err
	}
	return api.RuleTestResult{Expr: b.Expr, Level: b.Level, Entities: 3, Breaching: 1,
		Samples: []api.RuleTestSample{{Device: "d1", Value: 95, Breach: true}}}, nil
}

func (f *fakeRuleHost) Reserved(name string) bool { return name == "cpu_high" || name == "device_down" }

func (f *fakeRuleHost) Copy(_ context.Context, name string) (alertrule.Draft, string) {
	if name != "cpu_high" {
		return alertrule.Draft{}, "it divides two metrics"
	}
	return alertrule.Draft{Name: "cpu_high_custom", Kind: alertrule.KindMetric, Severity: "warning", For: 10 * time.Minute,
		Spec: alertrule.RuleSpec{Metric: &alertrule.MetricSpec{Metric: "hrProcessorLoad", Aggregate: "avg", Window: "15m", Op: ">", Threshold: 90}}}, ""
}

func (f *fakeRuleHost) Changed() { f.changes++ }

func ruleEnv(t *testing.T) (*env, *fakeRuleHost) {
	h, ph := newFakeRuleHost(), &fakePolicyHost{}
	e := newEnvDeps(t, 0, func(d *api.Deps) { d.AlertRules, d.AlertPolicy = h, ph })
	h.st = e.st
	return e, h
}

const cpuRule = `{"name":"cpu_server","kind":"metric","severity":"warning","for":"15m",
"spec":{"metric":{"metric":"hrProcessorLoad","aggregate":"max","window":"10m","op":">=","threshold":80}}}`

func createRule(t *testing.T, e *env, body string) api.AlertRuleInfo {
	t.Helper()
	rec := e.do("POST", "/api/v1/alert-rules", body)
	if rec.Code != http.StatusCreated {
		t.Fatalf("POST: %d %s", rec.Code, rec.Body)
	}
	var v api.AlertRuleInfo
	if err := json.Unmarshal(rec.Body.Bytes(), &v); err != nil || v.ID == "" {
		t.Fatalf("created = %s (%v)", rec.Body, err)
	}
	return v
}

func TestCustomRuleCRUDIsAuditedAndTellsTheRegistry(t *testing.T) {
	e, host := ruleEnv(t)
	v := createRule(t, e, cpuRule)
	if v.Kind != "metric" || !v.Editable || v.PolicyCount != 0 {
		t.Fatalf("created = %+v", v)
	}
	isProblem(t, e.do("POST", "/api/v1/alert-rules", cpuRule), http.StatusConflict)
	upd := strings.Replace(cpuRule, `"threshold":80`, `"threshold":85`, 1)
	if rec := e.do("PUT", "/api/v1/alert-rules/"+v.ID, upd); rec.Code != http.StatusOK {
		t.Fatalf("PUT: %d %s", rec.Code, rec.Body)
	}
	var got api.AlertRuleInfo
	e.getJSON(t, "/api/v1/alert-rules/"+v.ID, &got)
	if got.Spec == nil || got.Spec.Metric.Threshold != 85 {
		t.Fatalf("after PUT: %+v", got)
	}
	row, _ := e.st.GetAlertRule(t.Context(), v.ID)
	if row.ForSeconds != 900 || !row.Enabled {
		t.Fatalf("row = %+v", row)
	}
	if rec := e.do("DELETE", "/api/v1/alert-rules/"+v.ID, ""); rec.Code != http.StatusNoContent {
		t.Fatalf("DELETE: %d %s", rec.Code, rec.Body)
	}
	isProblem(t, e.do("GET", "/api/v1/alert-rules/"+v.ID, ""), http.StatusNotFound)
	if host.changes != 3 {
		t.Errorf("registry told %d times, want 3", host.changes)
	}
	audit, _, _ := e.st.ListAudit(t.Context(), store.Page{})
	if len(audit) < 3 || audit[0].Action != "alert_rule.delete" {
		t.Fatalf("audit = %+v", audit)
	}
}

func TestBuilderSpecWithUnknownMetricOrFieldIs422(t *testing.T) {
	e, _ := ruleEnv(t)
	for name, body := range map[string]string{
		"unknown metric": `{"name":"a","kind":"metric","severity":"info","spec":{"metric":{"metric":"nope","op":">","threshold":1}}}`,
		"hostile metric": `{"name":"a","kind":"metric","severity":"info","spec":{"metric":{"metric":"x\"} or vector(1)","op":">","threshold":1}}}`,
		"bad op":         `{"name":"a","kind":"metric","severity":"info","spec":{"metric":{"metric":"memTotalReal","op":"=~","threshold":1}}}`,
		"unknown field":  `{"name":"a","kind":"inventory","severity":"info","spec":{"inventory":{"conditions":[{"field":"password","op":"empty"}]}}}`,
		"wrong op type":  `{"name":"a","kind":"inventory","severity":"info","spec":{"inventory":{"conditions":[{"field":"site","op":">","value":"1"}]}}}`,
		"bad regex":      `{"name":"a","kind":"inventory","severity":"info","spec":{"inventory":{"conditions":[{"field":"site","op":"matches","value":"("}]}}}`,
		"bad template":   `{"name":"a","kind":"metric","severity":"info","summary":"{{.Nope}}","spec":{"metric":{"metric":"memTotalReal","op":">","threshold":1}}}`,
		"bad for":        `{"name":"a","kind":"metric","severity":"info","for":"soon","spec":{"metric":{"metric":"memTotalReal","op":">","threshold":1}}}`,
		"kind mismatch":  `{"name":"a","kind":"inventory","severity":"info","spec":{"metric":{"metric":"memTotalReal","op":">","threshold":1}}}`,
	} {
		if rec := e.do("POST", "/api/v1/alert-rules", body); rec.Code != http.StatusUnprocessableEntity {
			t.Errorf("%s: %d %s", name, rec.Code, rec.Body)
		}
	}
	if rules := listRules(t, e); len(rules) != 1 {
		t.Fatalf("a refused rule was stored: %+v", rules)
	}
}

func listRules(t *testing.T, e *env) []api.AlertRuleInfo {
	t.Helper()
	var l api.AlertRuleList
	e.getJSON(t, "/api/v1/alert-rules", &l)
	return l.Items
}

func TestBuiltinRulesCannotBeEditedOrDeleted(t *testing.T) {
	e, _ := ruleEnv(t)
	isProblem(t, e.do("PUT", "/api/v1/alert-rules/cpu_high", cpuRule), http.StatusConflict)
	isProblem(t, e.do("DELETE", "/api/v1/alert-rules/cpu_high", ""), http.StatusConflict)
	reserved := strings.Replace(cpuRule, "cpu_server", "cpu_high", 1)
	isProblem(t, e.do("POST", "/api/v1/alert-rules", reserved), http.StatusConflict)
	var got api.AlertRuleInfo
	e.getJSON(t, "/api/v1/alert-rules/cpu_high", &got)
	if got.Editable || got.Kind != "builtin" {
		t.Fatalf("builtin = %+v", got)
	}
}

func TestDeletingOrReshapingAReferencedRuleIs409(t *testing.T) {
	e, _ := ruleEnv(t)
	v := createRule(t, e, cpuRule)
	p := &store.AlertPolicy{Name: "servers", Rule: "cpu_server", Action: "alert", Enabled: true, Scope: `{"all":true}`}
	if err := e.st.CreateAlertPolicy(t.Context(), p); err != nil {
		t.Fatal(err)
	}
	rec := e.do("DELETE", "/api/v1/alert-rules/"+v.ID, "")
	isProblem(t, rec, http.StatusConflict)
	if !strings.Contains(rec.Body.String(), "servers") {
		t.Fatalf("409 does not name the policy: %s", rec.Body)
	}
	renamed := strings.Replace(cpuRule, "cpu_server", "cpu_renamed", 1)
	isProblem(t, e.do("PUT", "/api/v1/alert-rules/"+v.ID, renamed), http.StatusConflict)
	// Turning it into an interface rule would invalidate policies written for a device rule.
	inv := `{"name":"cpu_server","kind":"inventory","severity":"warning","spec":{"inventory":{"conditions":[{"field":"if_alias","op":"empty"}]}}}`
	isProblem(t, e.do("PUT", "/api/v1/alert-rules/"+v.ID, inv), http.StatusConflict)
	if err := e.st.DeleteAlertPolicy(t.Context(), p.ID); err != nil {
		t.Fatal(err)
	}
	if rec := e.do("DELETE", "/api/v1/alert-rules/"+v.ID, ""); rec.Code != http.StatusNoContent {
		t.Fatalf("DELETE after the policy is gone: %d %s", rec.Code, rec.Body)
	}
}

func TestCopyBuiltinMakesACustomRuleOrExplains(t *testing.T) {
	e, _ := ruleEnv(t)
	rec := e.do("POST", "/api/v1/alert-rules/cpu_high/copy", `{}`)
	if rec.Code != http.StatusCreated {
		t.Fatalf("copy: %d %s", rec.Code, rec.Body)
	}
	var v api.AlertRuleInfo
	if err := json.Unmarshal(rec.Body.Bytes(), &v); err != nil || v.Name != "cpu_high_custom" || !v.Editable {
		t.Fatalf("copy = %s", rec.Body)
	}
	isProblem(t, e.do("POST", "/api/v1/alert-rules/cpu_high/copy", `{}`), http.StatusConflict)
	if rec := e.do("POST", "/api/v1/alert-rules/cpu_high/copy", `{"name":"mine"}`); rec.Code != http.StatusCreated {
		t.Fatalf("named copy: %d %s", rec.Code, rec.Body)
	}
	rec = e.do("POST", "/api/v1/alert-rules/disk_full/copy", `{}`)
	if rec.Code != http.StatusNotFound { // the fake lists only cpu_high
		t.Fatalf("unknown: %d %s", rec.Code, rec.Body)
	}
	isProblem(t, e.do("POST", "/api/v1/alert-rules/"+v.ID+"/copy", `{}`), http.StatusConflict)
}

func TestCatalogueAndTestRoutes(t *testing.T) {
	e, _ := ruleEnv(t)
	var c api.RuleCatalogue
	e.getJSON(t, "/api/v1/alert-rules/catalogue", &c)
	if len(c.Metrics) != 2 || len(c.Fields) == 0 || len(c.Ops) != 6 {
		t.Fatalf("catalogue = %+v", c)
	}
	rec := e.do("POST", "/api/v1/alert-rules/test", cpuRule)
	var res api.RuleTestResult
	if err := json.Unmarshal(rec.Body.Bytes(), &res); rec.Code != http.StatusOK || err != nil || res.Breaching != 1 ||
		res.Expr != `max by (device) (last_over_time(hrProcessorLoad[10m]))` {
		t.Fatalf("test = %d %s", rec.Code, rec.Body)
	}
	isProblem(t, e.do("POST", "/api/v1/alert-rules/test", strings.Replace(cpuRule, "hrProcessorLoad", "nope", 1)), http.StatusUnprocessableEntity)
	if rules := listRules(t, e); len(rules) != 1 {
		t.Fatalf("a test stored a rule: %+v", rules)
	}
}

func TestAlertRuleRoutesAnswer503WithoutARegistry(t *testing.T) {
	e := newEnvDeps(t, 0, nil)
	isProblem(t, e.do("GET", "/api/v1/alert-rules/catalogue", ""), http.StatusServiceUnavailable)
	isProblem(t, e.do("POST", "/api/v1/alert-rules", cpuRule), http.StatusServiceUnavailable)
}