The owner wants LibreNMS-like 5-minute polling by default and the option
to poll a given device faster or slower. Devices gain poll_interval_s
(new migration; 0 means the default, otherwise 30 to 3600 s), settable in
the device edit dialog and through PUT /api/v1/devices/{slug}; the
poller picks it up on the next inventory refresh.
Everything that assumed one-minute samples now follows the device's
interval: graph and p95 rate windows are at least two polls, the max
series uses irate, the up-series lookback, the no_data warning, the
"unheard of for three polls" fallback and the rollup's thin-window check.
interface_errors widens to 10 m. profiletest keeps 60 s because the
expect budgets are keyed by the 1m group. Recorded as DEVIATIONS 49.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017TuKgyKve2XLFtrrXWMi2w
120 lines
3.7 KiB
Go
120 lines
3.7 KiB
Go
package rollup
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
"github.com/v10networks/nmon/internal/longq"
|
|
"github.com/v10networks/nmon/internal/metrics"
|
|
"github.com/v10networks/nmon/internal/store"
|
|
)
|
|
|
|
// Directions as written in the dir label.
|
|
const (
|
|
dirIn = "in"
|
|
dirOut = "out"
|
|
)
|
|
|
|
// counterMeta says which direction a raw octet counter feeds and whether it
|
|
// is 64-bit. Interfaces with HC counters use only the HC names; the others
|
|
// only the 32-bit names.
|
|
var counterMeta = map[string]struct {
|
|
dir string
|
|
wide bool
|
|
}{
|
|
"ifHCInOctets": {dirIn, true}, "ifHCOutOctets": {dirOut, true},
|
|
"ifInOctets": {dirIn, false}, "ifOutOctets": {dirOut, false},
|
|
}
|
|
|
|
// lookback is how far before a window the newest sample may be and still serve
|
|
// as the baseline of the window's first increase.
|
|
const lookback = 5 * time.Minute
|
|
|
|
// fiveMinute computes, writes and records the window starting at ws.
|
|
func (j *Job) fiveMinute(ctx context.Context, devs []deviceTarget, ws time.Time, force bool) error {
|
|
res, err := j.collectFiveMinute(ctx, devs, ws)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := j.commit(ctx, store.ResolutionFiveMinute, ws, res, force); err != nil {
|
|
return err
|
|
}
|
|
if end := ws.Add(FiveMinutes); end.After(j.lastDoneEnd) {
|
|
j.lastDoneEnd = end
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (j *Job) collectFiveMinute(ctx context.Context, devs []deviceTarget, ws time.Time) (windowResult, error) {
|
|
var out windowResult
|
|
var present, thin int
|
|
for _, d := range devs {
|
|
if len(d.ifaces) == 0 {
|
|
continue
|
|
}
|
|
if err := j.yield(ctx); err != nil {
|
|
return out, err
|
|
}
|
|
s, p, t, err := j.deviceRates(ctx, d, ws)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
out.samples = append(out.samples, s...)
|
|
present += p
|
|
thin += t
|
|
}
|
|
g, err := j.gaugeAverages(ctx, ws)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
out.samples = append(out.samples, g...)
|
|
sort.Slice(out.samples, func(a, b int) bool { return out.samples[a].SeriesKey() < out.samples[b].SeriesKey() })
|
|
out.partial = present == 0 || float64(thin) > j.o.PartialFraction*float64(present)
|
|
return out, nil
|
|
}
|
|
|
|
// deviceRates returns the rate samples of one device, how many interface
|
|
// directions had data and how many of those were thin (fewer samples than the
|
|
// poll interval predicts).
|
|
func (j *Job) deviceRates(ctx context.Context, d deviceTarget, ws time.Time) (out []metrics.Sample, present, thin int, err error) {
|
|
we := ws.Add(FiveMinutes)
|
|
// The baseline sample of a window's first increase may be one poll earlier than the window.
|
|
back := max(lookback, 2*d.every)
|
|
q := fmt.Sprintf(`{__name__=~"ifHCInOctets|ifHCOutOctets|ifInOctets|ifOutOctets",device=%q}[%ds]`,
|
|
d.slug, int((FiveMinutes+back)/time.Second))
|
|
body, err := j.o.Query.Query(ctx, "hot", q, we)
|
|
if err != nil {
|
|
return nil, 0, 0, fmt.Errorf("hot query for %s: %w", d.slug, err)
|
|
}
|
|
rows, err := parseResult(body)
|
|
if err != nil {
|
|
return nil, 0, 0, err
|
|
}
|
|
// A window is thin when it holds under 80 % of the samples the device's cadence predicts. At a cadence of
|
|
// 5 min or slower the prediction is at most one sample, so minInside is 0: a window with no sample at all is
|
|
// already skipped (not present), and a window with its one sample is complete, never thin.
|
|
minInside := int(float64(FiveMinutes/d.every) * 0.8)
|
|
for _, r := range rows {
|
|
meta, ok := counterMeta[r.labels["__name__"]]
|
|
ifc, known := d.ifaces[r.labels["iface"]]
|
|
if !ok || !known || ifc.HasHCCounters != meta.wide {
|
|
continue
|
|
}
|
|
cw, ok := measureCounter(r.pts, ws.UnixMilli(), we.UnixMilli(), meta.wide)
|
|
if cw.inside == 0 {
|
|
continue
|
|
}
|
|
present++
|
|
if cw.inside < minInside {
|
|
thin++
|
|
}
|
|
if ok {
|
|
out = append(out, metrics.New(longq.MetricRate5m, cw.rateBps, ws,
|
|
metrics.LabelDevice, d.slug, metrics.LabelIface, ifc.ID, metrics.LabelDir, meta.dir))
|
|
}
|
|
}
|
|
return out, present, thin, nil
|
|
}
|