Files
nmon/internal/app/diskwire.go

140 lines
4.2 KiB
Go

package app
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"sort"
"strings"
"time"
"github.com/v10networks/nmon/internal/clock"
"github.com/v10networks/nmon/internal/diskguard"
)
// diskAlarmTTL is how long a disk alarm stays open after the condition was last seen: the guard raises on the
// edge only, so diskLoop re-raises every minute while the condition holds and the alarm resolves itself after.
const (
diskAlarmTTL = 3 * time.Minute
diskLoopEvery = time.Minute
warmWindow = 15 * time.Minute
warmTimeout = time.Minute
maxWarmSeries = 2_000_000
)
func diskTTL(string) time.Duration { return diskAlarmTTL }
// startDisk starts the disk guard and puts its admission policy in front of the hot write queue. The poller,
// push ingest and the checks write through a.hotW; the rollup writes to the long store and pauses with the guard.
func (a *App) startDisk(s *settings) error {
g, err := diskguard.New(diskguard.Options{
Path: s.DataDir, Alarms: kindAlarms{a.alerting.sys, diskTTL},
AlertUsed: s.cfg.Disk.AlertUsed, AdmitUsed: s.cfg.Disk.AdmitUsed, ReserveFree: s.cfg.Disk.ReserveFree,
})
if err != nil {
return fmt.Errorf("disk guard: %w", err)
}
adm := diskguard.NewAdmission(a.hot, g, 0, nil)
a.guard, a.hotW = g, adm
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{}, 2)
go func() { g.Run(ctx); done <- struct{}{} }()
go func() { a.diskLoop(ctx); done <- struct{}{} }()
go a.warmAdmission(ctx, adm)
a.onStop("disk guard", func(context.Context) error {
cancel()
<-done
<-done
return nil
})
return nil
}
// diskLoop keeps the three disk alarms open while their conditions hold.
func (a *App) diskLoop(ctx context.Context) {
t := clock.Real().NewTicker(diskLoopEvery)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C():
}
g := a.guard
detail := fmt.Sprintf("%.1f%% of %s used", g.Used()*100, a.dataDir)
for kind, on := range map[string]bool{
diskguard.AlarmNewSeriesHalt: g.RefuseNewSeries(),
diskguard.AlarmReserve: g.Throttle(),
} {
if on {
kindAlarms{a.alerting.sys, diskTTL}.Raise(kind, "", detail)
}
}
}
}
// warmAdmission tells the admission writer which series the hot store already has, so a restart while the disk is
// above 90% does not treat every existing series as new.
func (a *App) warmAdmission(ctx context.Context, adm *diskguard.Admission) {
wctx, cancel := context.WithTimeout(ctx, warmTimeout)
defer cancel()
keys, err := hotSeriesKeys(wctx, a.sup.HotURL(), clock.Real().Now())
if err != nil {
a.log.Warn("could not list existing series for the disk guard; new series are treated as new until written once", "err", err)
return
}
adm.Warm(keys)
a.log.Info("disk guard knows the hot store's series", "series", len(keys))
}
// hotSeriesKeys lists the series written recently as metrics.Sample.SeriesKey strings.
func hotSeriesKeys(ctx context.Context, base string, now time.Time) ([]string, error) {
q := url.Values{
"match[]": {`{__name__=~".+"}`}, "start": {fmt.Sprint(now.Add(-warmWindow).Unix())}, "end": {fmt.Sprint(now.Unix())},
"limit": {fmt.Sprint(maxWarmSeries)},
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/api/v1/series?"+q.Encode(), nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("series list: %s", resp.Status)
}
var body struct {
Data []map[string]string `json:"data"`
}
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
return nil, err
}
keys := make([]string, 0, len(body.Data))
for _, m := range body.Data {
keys = append(keys, seriesKeyOf(m))
}
return keys, nil
}
// seriesKeyOf builds the key metrics.Sample.SeriesKey would for these labels (sorted by name).
func seriesKeyOf(m map[string]string) string {
names := make([]string, 0, len(m))
for n := range m {
if n != "__name__" {
names = append(names, n)
}
}
sort.Strings(names)
var b strings.Builder
b.WriteString(m["__name__"])
for _, n := range names {
b.WriteByte(0)
b.WriteString(n + "=" + m[n])
}
return b.String()
}