Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
131 lines
4.5 KiB
Go
131 lines
4.5 KiB
Go
// Package notify delivers alert notifications. The alert_deliveries table is
|
|
// the queue: Enqueue renders and redacts a payload and stores it as a pending
|
|
// row, workers send it (webhook or SMTP) through the outbound policy, failures
|
|
// are retried with exponential backoff for an hour, and every attempt is a row
|
|
// in the same table. Pending rows are picked up again after a restart.
|
|
package notify
|
|
|
|
import (
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"time"
|
|
)
|
|
|
|
// Receiver kinds.
|
|
const (
|
|
KindWebhook = "webhook"
|
|
KindSMTP = "smtp"
|
|
)
|
|
|
|
// Notification states.
|
|
const (
|
|
StateFiring = "firing"
|
|
StateResolved = "resolved"
|
|
StateFlapping = "flapping"
|
|
StateStale = "stale"
|
|
StateSuppressedSummary = "suppressed_summary"
|
|
)
|
|
|
|
// Receiver is one configured destination.
|
|
type Receiver struct {
|
|
Name string `yaml:"name" json:"name"`
|
|
Kind string `yaml:"kind" json:"kind"` // webhook | smtp
|
|
// URL is the webhook endpoint. It may embed a token, so it is never logged.
|
|
URL string `yaml:"url" json:"url"`
|
|
// Headers are extra webhook request headers. Values may carry secrets
|
|
// (Authorization, X-Api-Key) and are redacted wherever they are logged.
|
|
Headers map[string]string `yaml:"headers" json:"headers"`
|
|
// Template is a Go text/template. For a webhook it must render a JSON
|
|
// document; for SMTP it renders "Subject: ...", a blank line and the body.
|
|
// Empty selects the default for the kind. Templates see only Notification.
|
|
Template string `yaml:"template" json:"template"`
|
|
SMTP SMTP `yaml:"smtp" json:"smtp"`
|
|
}
|
|
|
|
// SMTP configures an smtp receiver.
|
|
type SMTP struct {
|
|
Host string `yaml:"host" json:"host"`
|
|
Port int `yaml:"port" json:"port"` // 0: 587 with StartTLS, else 25
|
|
From string `yaml:"from" json:"from"`
|
|
To []string `yaml:"to" json:"to"`
|
|
Username string `yaml:"username" json:"username"`
|
|
Password string `yaml:"password" json:"password"`
|
|
StartTLS bool `yaml:"starttls" json:"starttls"`
|
|
}
|
|
|
|
// Redacted returns a copy that is safe to log: no URL, header values and
|
|
// SMTP password masked.
|
|
func (r Receiver) Redacted() Receiver {
|
|
r.Headers = RedactHeaders(r.Headers)
|
|
if r.URL != "" {
|
|
r.URL = Redacted
|
|
}
|
|
if r.SMTP.Password != "" {
|
|
r.SMTP.Password = Redacted
|
|
}
|
|
return r
|
|
}
|
|
|
|
// Notification is everything a template can see. It deliberately carries no
|
|
// configuration, credentials or receivers. The queue builds the template's
|
|
// copy field by field (see safeCopy).
|
|
type Notification struct {
|
|
// IncidentID is "<alert id>-<fired_at unix>" and identical on every retry
|
|
// and for the firing and resolved notifications of one incident.
|
|
IncidentID string `json:"incident_id"`
|
|
AlertID int64 `json:"alert_id"`
|
|
Rule string `json:"rule"`
|
|
State string `json:"state"`
|
|
Severity string `json:"severity"`
|
|
Device struct {
|
|
Slug string `json:"slug"`
|
|
Name string `json:"name"`
|
|
} `json:"device"`
|
|
Interface struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Alias string `json:"alias"`
|
|
} `json:"interface"`
|
|
Summary string `json:"summary"`
|
|
Description string `json:"description"`
|
|
FiredAt time.Time `json:"fired_at"`
|
|
ClearedAt *time.Time `json:"cleared_at"`
|
|
Labels map[string]string `json:"labels"`
|
|
Parents []string `json:"parents"`
|
|
SuppressedChildren int `json:"suppressed_children"`
|
|
URL string `json:"url"`
|
|
}
|
|
|
|
// IncidentID builds the stable incident identifier.
|
|
func IncidentID(alertID int64, firedAt time.Time) string {
|
|
return fmt.Sprintf("%d-%d", alertID, firedAt.Unix())
|
|
}
|
|
|
|
// safeCopy rebuilds n from an explicit list of fields. Adding a field to
|
|
// Notification therefore does not expose it to templates until it is added
|
|
// here. Label values under secret-looking keys are masked.
|
|
func (n Notification) safeCopy() Notification {
|
|
var c Notification
|
|
c.IncidentID, c.AlertID, c.Rule, c.State, c.Severity = n.IncidentID, n.AlertID, n.Rule, n.State, n.Severity
|
|
c.Device.Slug, c.Device.Name = n.Device.Slug, n.Device.Name
|
|
c.Interface.ID, c.Interface.Name, c.Interface.Alias = n.Interface.ID, n.Interface.Name, n.Interface.Alias
|
|
c.Summary, c.Description = n.Summary, n.Description
|
|
c.FiredAt = n.FiredAt.UTC()
|
|
if n.ClearedAt != nil {
|
|
t := n.ClearedAt.UTC()
|
|
c.ClearedAt = &t
|
|
}
|
|
c.Labels = make(map[string]string, len(n.Labels))
|
|
for k, v := range maps.All(n.Labels) {
|
|
if IsSensitiveKey(k) {
|
|
v = Redacted
|
|
}
|
|
c.Labels[k] = v
|
|
}
|
|
c.Parents = slices.Clone(n.Parents)
|
|
c.SuppressedChildren = n.SuppressedChildren
|
|
c.URL = n.URL
|
|
return c
|
|
}
|