Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
140 lines
4.2 KiB
Go
140 lines
4.2 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/v10networks/nmon/internal/clock"
|
|
"github.com/v10networks/nmon/internal/diskguard"
|
|
)
|
|
|
|
// diskAlarmTTL is how long a disk alarm stays open after the condition was last seen: the guard raises on the
|
|
// edge only, so diskLoop re-raises every minute while the condition holds and the alarm resolves itself after.
|
|
const (
|
|
diskAlarmTTL = 3 * time.Minute
|
|
diskLoopEvery = time.Minute
|
|
warmWindow = 15 * time.Minute
|
|
warmTimeout = time.Minute
|
|
maxWarmSeries = 2_000_000
|
|
)
|
|
|
|
func diskTTL(string) time.Duration { return diskAlarmTTL }
|
|
|
|
// startDisk starts the disk guard and puts its admission policy in front of the hot write queue. The poller,
|
|
// push ingest and the checks write through a.hotW; the rollup writes to the long store and pauses with the guard.
|
|
func (a *App) startDisk(s *settings) error {
|
|
g, err := diskguard.New(diskguard.Options{
|
|
Path: s.DataDir, Alarms: kindAlarms{a.alerting.sys, diskTTL},
|
|
AlertUsed: s.cfg.Disk.AlertUsed, AdmitUsed: s.cfg.Disk.AdmitUsed, ReserveFree: s.cfg.Disk.ReserveFree,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("disk guard: %w", err)
|
|
}
|
|
adm := diskguard.NewAdmission(a.hot, g, 0, nil)
|
|
a.guard, a.hotW = g, adm
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan struct{}, 2)
|
|
go func() { g.Run(ctx); done <- struct{}{} }()
|
|
go func() { a.diskLoop(ctx); done <- struct{}{} }()
|
|
go a.warmAdmission(ctx, adm)
|
|
a.onStop("disk guard", func(context.Context) error {
|
|
cancel()
|
|
<-done
|
|
<-done
|
|
return nil
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// diskLoop keeps the three disk alarms open while their conditions hold.
|
|
func (a *App) diskLoop(ctx context.Context) {
|
|
t := clock.Real().NewTicker(diskLoopEvery)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C():
|
|
}
|
|
g := a.guard
|
|
detail := fmt.Sprintf("%.1f%% of %s used", g.Used()*100, a.dataDir)
|
|
for kind, on := range map[string]bool{
|
|
diskguard.AlarmNewSeriesHalt: g.RefuseNewSeries(),
|
|
diskguard.AlarmReserve: g.Throttle(),
|
|
} {
|
|
if on {
|
|
kindAlarms{a.alerting.sys, diskTTL}.Raise(kind, "", detail)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// warmAdmission tells the admission writer which series the hot store already has, so a restart while the disk is
|
|
// above 90% does not treat every existing series as new.
|
|
func (a *App) warmAdmission(ctx context.Context, adm *diskguard.Admission) {
|
|
wctx, cancel := context.WithTimeout(ctx, warmTimeout)
|
|
defer cancel()
|
|
keys, err := hotSeriesKeys(wctx, a.sup.HotURL(), clock.Real().Now())
|
|
if err != nil {
|
|
a.log.Warn("could not list existing series for the disk guard; new series are treated as new until written once", "err", err)
|
|
return
|
|
}
|
|
adm.Warm(keys)
|
|
a.log.Info("disk guard knows the hot store's series", "series", len(keys))
|
|
}
|
|
|
|
// hotSeriesKeys lists the series written recently as metrics.Sample.SeriesKey strings.
|
|
func hotSeriesKeys(ctx context.Context, base string, now time.Time) ([]string, error) {
|
|
q := url.Values{
|
|
"match[]": {`{__name__=~".+"}`}, "start": {fmt.Sprint(now.Add(-warmWindow).Unix())}, "end": {fmt.Sprint(now.Unix())},
|
|
"limit": {fmt.Sprint(maxWarmSeries)},
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/api/v1/series?"+q.Encode(), nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = resp.Body.Close() }()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("series list: %s", resp.Status)
|
|
}
|
|
var body struct {
|
|
Data []map[string]string `json:"data"`
|
|
}
|
|
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
|
return nil, err
|
|
}
|
|
keys := make([]string, 0, len(body.Data))
|
|
for _, m := range body.Data {
|
|
keys = append(keys, seriesKeyOf(m))
|
|
}
|
|
return keys, nil
|
|
}
|
|
|
|
// seriesKeyOf builds the key metrics.Sample.SeriesKey would for these labels (sorted by name).
|
|
func seriesKeyOf(m map[string]string) string {
|
|
names := make([]string, 0, len(m))
|
|
for n := range m {
|
|
if n != "__name__" {
|
|
names = append(names, n)
|
|
}
|
|
}
|
|
sort.Strings(names)
|
|
var b strings.Builder
|
|
b.WriteString(m["__name__"])
|
|
for _, n := range names {
|
|
b.WriteByte(0)
|
|
b.WriteString(n + "=" + m[n])
|
|
}
|
|
return b.String()
|
|
}
|