jleung
b3aae800bf
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Lx9FNLG2CqeBbVGGd9bFsP
340 lines
25 KiB
Bash
Executable File
340 lines
25 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Gate M5: a replayed Proxmox sample shows under its node, and a 2 y query hits the 1 h tier.
|
|
# Procedure: docs/milestones/M5-ingest-rollup-checks.md.
|
|
# 1. make lint; unit tests of the packages this milestone adds.
|
|
# 2. make build + make devtools.
|
|
# 3. Mock Proxmox API (nmon-devtools proxmox-mock, testdata/proxmox/cluster-resources.json), an HTTP sink
|
|
# (check target), two simulated switches (testdata/topologies/m5.yaml). nmon serves with its outbound
|
|
# allow-list naming [::1] and a Proxmox cluster whose token secret is set through PUT /api/v1/secrets.
|
|
# 4. (a) Proxmox sync: node pve1 and its guests appear, guests hosted_on pve1; the secret is write-only.
|
|
# (b) a push token bound to pve1; testdata/push/proxmox.influx replayed (timestamps rebased to now) to
|
|
# POST /write; GET /api/v1/devices/pve1 lists the pushed series; Graphite works with a token line.
|
|
# Negatives: no token 401, reserved prefix dropped, a series cap raises an alert, revoke -> 401.
|
|
# (c) rollups: ~25 months of 1 h and 70 days of 5 m rollups written to the long store, raw history into hot,
|
|
# `nmon rollup repair` over the last 2 h; GET /api/v1/graph for 2 y has tiers 1h + raw, stitched_at set,
|
|
# long data with the exact backfilled values; last month's p95 and the graph's month p95 carry source.
|
|
# (d) a group of two interfaces: the group graph is the oriented sum; p95 for the group.
|
|
# (e) an HTTP check against a local server shows nmon_check_up.
|
|
# 5. openapi-check against m3 (additive only), contract drift, mock UI specs.
|
|
# 6. Live screenshots (groups, tokens, checks, a VM device, the 2 y stitched graph) into docs/screenshots/M5/.
|
|
# Needs: go, make, node 22+, npm, Playwright's chromium (cd web && npx playwright install chromium), curl, ss, awk, python3, date (GNU).
|
|
set -uo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
# Maintainability is a hard rule (docs/MAINTAINABILITY.md): no gate passes with lint findings.
|
|
make lint || { echo "FAIL: make lint"; exit 1; }
|
|
|
|
TMP="$(mktemp -d)"
|
|
FAILS=0
|
|
NMON_PID="" SINK_PID="" AGENTS_PID="" PVE_PID=""
|
|
SHOTS="$ROOT/${NMON_SHOT_DIR:-docs/screenshots/M5}"
|
|
TOPO="$ROOT/testdata/topologies/m5.yaml"
|
|
DEVTOOLS="$ROOT/nmon-devtools"
|
|
MAP="$TMP/slugs.map"
|
|
CTL="$TMP/ctl"
|
|
|
|
pass() { echo "PASS: $*"; }
|
|
fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
|
|
die() { fail "$*"; exit 1; }
|
|
|
|
cleanup() {
|
|
for p in "$NMON_PID" "$AGENTS_PID" "$SINK_PID" "$PVE_PID"; do
|
|
[ -n "$p" ] && kill -TERM "$p" 2>/dev/null && wait "$p" 2>/dev/null
|
|
done
|
|
rm -rf "$TMP"
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT TERM
|
|
|
|
pick_port() { # a port free for both TCP and UDP
|
|
local p
|
|
while :; do
|
|
p=$((20000 + RANDOM % 30000))
|
|
ss -H -lntu 2>/dev/null | awk '{print $5}' | grep -q "[:.]$p\$" || { echo "$p"; return; }
|
|
done
|
|
}
|
|
|
|
pick_udp_range() { # $1 consecutive UDP ports that are all free
|
|
local base n="$1" i busy
|
|
while :; do
|
|
base=$((20000 + RANDOM % 25000)); busy=0
|
|
for i in $(seq 0 $((n - 1))); do
|
|
ss -H -lnu 2>/dev/null | awk '{print $5}' | grep -q "[:.]$((base + i))\$" && { busy=1; break; }
|
|
done
|
|
[ "$busy" -eq 0 ] && { echo "$base"; return; }
|
|
done
|
|
}
|
|
|
|
wait_for() { # wait_for <seconds> <description> <command...>
|
|
local n="$1" what="$2"; shift 2
|
|
for _ in $(seq 1 "$((n * 2))"); do "$@" && return 0; sleep 0.5; done
|
|
echo "timed out after ${n}s waiting for $what"; return 1
|
|
}
|
|
|
|
# jq-less JSON access: py <python expression over d> reads JSON on stdin.
|
|
py() { python3 -I -c 'import json,sys; d=json.load(sys.stdin); print(eval(sys.argv[1]))' "$1"; }
|
|
api() { curl -fsS "$URL$1"; }
|
|
api_up() { curl -fsS "$URL/api/v1/devices" >/dev/null 2>&1; }
|
|
slug_by_name() { api "/api/v1/devices?per_page=1000" | py "next((i['slug'] for i in d['items'] if i['name']=='$1'), '')" 2>/dev/null; }
|
|
# near <value> <want> <relative tolerance>
|
|
near() { python3 -I -c 'import sys; v,w,t=map(float,sys.argv[1:4]); sys.exit(0 if abs(v-w)<=abs(w)*t else 1)' "$1" "$2" "$3"; }
|
|
vm_value() { # vm_value <base url> <promql> [time]: first sample value or empty
|
|
curl -fsS -G "$1/api/v1/query" --data-urlencode "query=$2" ${3:+--data-urlencode "time=$3"} | py "d['data']['result'][0]['value'][1] if d['data']['result'] else ''"
|
|
}
|
|
|
|
# 1. Unit tests of what this milestone adds.
|
|
echo "== unit tests"
|
|
if go test -race -count=1 ./internal/longq/... ./internal/ingest/... ./internal/rollup/... ./internal/p95/... ./internal/groups/... \
|
|
./internal/proxmox/... ./internal/checks/... ./internal/api/... ./internal/app/... ./internal/config/... ./internal/alert/... \
|
|
./internal/store/... ./cmd/... >"$TMP/unit.log" 2>&1; then
|
|
pass "go test -race: longq, ingest, rollup, p95, groups, proxmox, checks, api, app, config, alert, store, cmd"
|
|
else
|
|
tail -n 30 "$TMP/unit.log"; fail "go test -race"
|
|
fi
|
|
if go test -count=1 -tags 'integration vmembed' ./internal/rollup/... ./internal/p95/... ./internal/groups/... ./internal/api/... \
|
|
>"$TMP/integ.log" 2>&1; then
|
|
pass "integration tests against a real VictoriaMetrics: rollup, p95, groups, api (highest v wins, no lookback leak)"
|
|
else
|
|
tail -n 30 "$TMP/integ.log"; fail "integration tests"
|
|
fi
|
|
|
|
# 2. Binaries.
|
|
echo "== make build"
|
|
make build >"$TMP/build.log" 2>&1 || { tail -n 30 "$TMP/build.log"; die "make build"; }
|
|
make devtools >"$TMP/devtools.log" 2>&1 || { tail -n 30 "$TMP/devtools.log"; die "make devtools"; }
|
|
pass "nmon and nmon-devtools built"
|
|
|
|
# 3. Mock Proxmox, check target, agents, nmon.
|
|
echo "== mock proxmox + sink + agents + nmon serve"
|
|
mkdir -p "$CTL"; : >"$TMP/sink.jsonl"
|
|
PVE_SECRET="gate-pve-token-secret"
|
|
"$DEVTOOLS" proxmox-mock --file testdata/proxmox/cluster-resources.json --secret "$PVE_SECRET" --port-file "$TMP/pve.port" >"$TMP/pve.log" 2>&1 &
|
|
PVE_PID=$!
|
|
"$DEVTOOLS" webhook-sink --listen '[::1]:0' --out "$TMP/sink.jsonl" --port-file "$TMP/sink.port" >"$TMP/sink.log" 2>&1 &
|
|
SINK_PID=$!
|
|
wait_for 20 "mock proxmox + sink" test -s "$TMP/pve.port" -a -s "$TMP/sink.port" || die "mock servers did not start"
|
|
PVE_PORT="$(cat "$TMP/pve.port")"; SINK_PORT="$(cat "$TMP/sink.port")"
|
|
NDEV="$(grep -c '^ - {slug:' "$TOPO")"
|
|
BASE="$(pick_udp_range "$NDEV")"
|
|
"$DEVTOOLS" topology-agents --topology "$TOPO" --port-base "$BASE" --control-dir "$CTL" >"$TMP/agents.log" 2>&1 &
|
|
AGENTS_PID=$!
|
|
wait_for 30 "the simulated agents" grep -q 'agents ready' "$TMP/agents.log" || { cat "$TMP/agents.log"; die "agents did not start"; }
|
|
|
|
PORT="$(pick_port)"; GPORT="$(pick_port)"
|
|
URL="http://127.0.0.1:$PORT"
|
|
DATA="$TMP/data"
|
|
mkdir -p "$DATA"
|
|
cat >"$DATA/config.yaml" <<YAML
|
|
alerting:
|
|
outbound_allow: ["::1"]
|
|
proxmox:
|
|
clusters:
|
|
- name: main
|
|
url: http://[::1]:$PVE_PORT
|
|
token_id: root@pam!nmon
|
|
secret_name: pve:main
|
|
checks:
|
|
tick: 2s
|
|
YAML
|
|
./nmon serve --data-dir "$DATA" --secret-key-file "$TMP/secret.key" --insecure-http --listen "127.0.0.1:$PORT" \
|
|
--poll-interval 5s --icmp off --trap-listen '' --graphite-listen "127.0.0.1:$GPORT" >"$TMP/nmon.log" 2>&1 &
|
|
NMON_PID=$!
|
|
wait_for 90 "nmon to serve" api_up || { tail -n 20 "$TMP/nmon.log"; die "nmon serve did not start"; }
|
|
HOT="$(grep -o 'hot=http://[^ ]*' "$TMP/nmon.log" | head -n 1 | cut -d= -f2)"
|
|
LONG="$(grep -o 'long=http://[^ ]*' "$TMP/nmon.log" | head -n 1 | cut -d= -f2)"
|
|
[ -n "$HOT" ] && [ -n "$LONG" ] || die "cannot read the metric store URLs from the log"
|
|
"$DEVTOOLS" topology-add --api "$URL" --topology "$TOPO" --port-base "$BASE" --map-out "$MAP" >"$TMP/add.log" 2>&1 || { cat "$TMP/add.log"; die "topology-add"; }
|
|
pass "nmon up; $(cat "$TMP/add.log")"
|
|
grep -q 'graphite listener up' "$TMP/nmon.log" && pass "the Graphite listener is up on 127.0.0.1:$GPORT" || fail "no Graphite listener"
|
|
SW1="$(awk '$2=="sw-1" {print $1}' "$MAP")"; SW2="$(awk '$2=="sw-2" {print $1}' "$MAP")"
|
|
have_ifaces() { [ "$(api "/api/v1/devices/$SW1" | py "len(d['interfaces'])")" -ge 1 ] && [ "$(api "/api/v1/devices/$SW2" | py "len(d['interfaces'])")" -ge 1 ]; }
|
|
wait_for 90 "interfaces on both switches" have_ifaces || die "switches have no interfaces"
|
|
IFA="$(api "/api/v1/devices/$SW1" | py "d['interfaces'][0]['id']")"
|
|
IFB="$(api "/api/v1/devices/$SW2" | py "d['interfaces'][0]['id']")"
|
|
|
|
# 4a. Proxmox.
|
|
echo "== (a) proxmox"
|
|
CODE="$(curl -s -o "$TMP/secret.json" -w '%{http_code}' -X PUT "$URL/api/v1/secrets/pve:main" -H 'Content-Type: application/json' -d "{\"value\":\"$PVE_SECRET\"}")"
|
|
[ "$CODE" = 204 ] && pass "PUT /api/v1/secrets/pve:main stores the token secret" || { cat "$TMP/secret.json"; fail "PUT secret: HTTP $CODE"; }
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' "$URL/api/v1/secrets/pve:main")" != 200 ] && pass "a secret cannot be read back" || fail "GET /secrets/{name} answered 200"
|
|
curl -s -X POST "$URL/api/v1/proxmox/main/sync" -o /dev/null
|
|
node_up() { [ -n "$(slug_by_name pve1)" ]; }
|
|
wait_for 60 "node pve1 to appear" node_up || { api /api/v1/proxmox; tail -n 5 "$TMP/nmon.log"; die "Proxmox sync did not create pve1"; }
|
|
PVE1="$(slug_by_name pve1)"
|
|
api "/api/v1/devices/$PVE1" >"$TMP/pve1.json"
|
|
[ "$(py "d['kind']" <"$TMP/pve1.json")" = check_only ] && pass "node pve1 is a check_only device ($PVE1)" || fail "pve1 kind: $(py "d['kind']" <"$TMP/pve1.json")"
|
|
NG="$(py "len(d['guests'])" <"$TMP/pve1.json")"
|
|
[ "$NG" -ge 3 ] && pass "pve1 lists $NG guests hosted on it" || { cat "$TMP/pve1.json"; fail "pve1 guests: $NG"; }
|
|
GUEST="$(py "d['guests'][0]['slug']" <"$TMP/pve1.json")"
|
|
[ "$(api "/api/v1/devices/$GUEST" | py "d['hosted_on']['slug']")" = "$PVE1" ] && pass "guest $GUEST is hosted_on pve1" || fail "guest hosted_on"
|
|
api /api/v1/proxmox | grep -q '"name":"main"' && api /api/v1/proxmox | grep -q '"last_sync"' && pass "GET /api/v1/proxmox reports the cluster's last sync" || fail "GET /api/v1/proxmox"
|
|
node_reach() { [ "$(api "/api/v1/devices/$PVE1" | py "d['reachability']")" = up ]; }
|
|
wait_for 30 "pve1 to read up" node_reach && pass "pve1 reads up from the node status" || fail "pve1 reachability: $(api "/api/v1/devices/$PVE1" | py "d['reachability']")"
|
|
! grep -q "$PVE_SECRET" "$TMP/data/config.yaml" "$TMP/nmon.log" && pass "the token secret is in neither config.yaml nor the log" || fail "token secret leaked"
|
|
|
|
# 4b. Push.
|
|
echo "== (b) push ingest"
|
|
CODE="$(curl -s -o "$TMP/tok.json" -w '%{http_code}' -X POST "$URL/api/v1/tokens" -H 'Content-Type: application/json' \
|
|
-d "{\"name\":\"gate-pve1\",\"scope\":\"push\",\"devices\":[\"$PVE1\"]}")"
|
|
[ "$CODE" = 201 ] && pass "minted a push token bound to pve1" || { cat "$TMP/tok.json"; die "mint token: HTTP $CODE"; }
|
|
SECRET="$(py "d['secret']" <"$TMP/tok.json")"; TOKID="$(py "d['id']" <"$TMP/tok.json")"
|
|
case "$SECRET" in nmp_"$TOKID"_*) pass "token format nmp_<id>_<secret>";; *) fail "token format: $SECRET";; esac
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$URL/api/v1/tokens" -H 'Content-Type: application/json' -d "{\"name\":\"x\",\"scope\":\"admin\",\"devices\":[\"$PVE1\"]}")" = 422 ] &&
|
|
pass "a scope other than push is refused for now" || fail "non-push scope accepted"
|
|
api /api/v1/tokens >"$TMP/toks.json"
|
|
! grep -q "$SECRET" "$TMP/toks.json" && ! grep -q '\$' "$TMP/toks.json" && grep -q '"gate-pve1"' "$TMP/toks.json" &&
|
|
pass "GET /api/v1/tokens lists the token without its secret or hash" || { cat "$TMP/toks.json"; fail "token listing"; }
|
|
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$URL/write" --data-binary 'cpu,host=pve1 v=1')" = 401 ] && pass "POST /write without a token is 401" || fail "unauthenticated write"
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$URL/write" -H 'Authorization: Bearer nmp_999_notatoken' --data-binary 'cpu v=1')" = 401 ] && pass "a made-up token is 401" || fail "bad token"
|
|
"$DEVTOOLS" push-replay --url "$URL/write" --token "$SECRET" --file testdata/push/proxmox.influx \
|
|
--append 'nmon_evil,host=pve1 v=1' >"$TMP/replay.log" 2>&1 && pass "replayed testdata/push/proxmox.influx: $(cat "$TMP/replay.log")" || { cat "$TMP/replay.log"; fail "replay"; }
|
|
pushed() { [ "$(api "/api/v1/devices/$PVE1" | py "d['pushed_series']['count']")" -ge 5 ]; }
|
|
wait_for 60 "pushed series on pve1" pushed || { api "/api/v1/devices/$PVE1" | py "d['pushed_series']"; fail "no pushed series under pve1"; }
|
|
api "/api/v1/devices/$PVE1" | py "' '.join(d['pushed_series']['names'])" >"$TMP/names.txt"
|
|
grep -q 'memory_memtotal\|memtotal' "$TMP/names.txt" && pass "GET /api/v1/devices/pve1 lists $(api "/api/v1/devices/$PVE1" | py "d['pushed_series']['count']") pushed series under the node" ||
|
|
{ cat "$TMP/names.txt"; fail "pushed series names"; }
|
|
grep -q 'nmon_evil' "$TMP/names.txt" && fail "a reserved nmon_ metric was stored" || pass "the reserved nmon_ prefix was dropped"
|
|
curl -fsS "$URL/metrics" | grep -q 'nmon_ingest_dropped_total{reason="reserved_name"} [1-9]' && pass "the drop is counted in nmon_ingest_dropped_total" || fail "no reserved_name drop counter"
|
|
[ "$(vm_value "$HOT" 'count({device="'"$PVE1"'",push="1"})')" != "" ] && pass "pushed samples carry the token's device label" || fail "no series for device=$PVE1 in hot"
|
|
[ "$(vm_value "$HOT" 'count({push="1",device!="'"$PVE1"'"})')" = "" ] && pass "no pushed series under any other device (the token binds identity)" || fail "pushed series for another device"
|
|
|
|
# Graphite: token as the first line.
|
|
{ exec 3<>"/dev/tcp/127.0.0.1/$GPORT" && printf 'token %s\ngate.graphite.metric 42 %s\n' "$SECRET" "$(date +%s)" >&3; exec 3>&-; } 2>/dev/null
|
|
graphite() { api "/api/v1/devices/$PVE1" | py "' '.join(d['pushed_series']['names'])" | grep -q graphite; }
|
|
wait_for 40 "the Graphite sample" graphite && pass "Graphite plaintext with a token line is stored under pve1" || fail "Graphite sample not stored"
|
|
|
|
# Series cap raises an alert.
|
|
CODE="$(curl -s -o "$TMP/tok2.json" -w '%{http_code}' -X POST "$URL/api/v1/tokens" -H 'Content-Type: application/json' \
|
|
-d "{\"name\":\"gate-capped\",\"devices\":[\"$PVE1\"],\"series_cap\":2}")"
|
|
CAPSECRET="$(py "d['secret']" <"$TMP/tok2.json")"
|
|
printf 'capm,host=pve1 a=1,b=2,c=3,d=4,e=5\n' | curl -s -o /dev/null -X POST "$URL/write" -H "Authorization: Bearer $CAPSECRET" --data-binary @-
|
|
cap_alert() { api "/api/v1/alerts" | grep -q '"rule":"ingest_series_cap"'; }
|
|
wait_for 30 "the ingest_series_cap alert" cap_alert && pass "a series cap raises an ingest_series_cap alert through the alerting pipeline" || { api /api/v1/alerts; fail "no ingest_series_cap alert"; }
|
|
CAPCOUNT="$(api "/api/v1/alerts" | grep -o '"rule":"ingest_series_cap"' | wc -l)"
|
|
printf 'capm,host=pve1 f=1,g=2,h=3\n' | curl -s -o /dev/null -X POST "$URL/write" -H "Authorization: Bearer $CAPSECRET" --data-binary @-
|
|
sleep 2
|
|
[ "$(api "/api/v1/alerts" | grep -o '"rule":"ingest_series_cap"' | wc -l)" -eq "$CAPCOUNT" ] && pass "the alarm is deduplicated across repeats" || fail "ingest alarm duplicated"
|
|
|
|
# Revocation.
|
|
CAPID="$(py "d['id']" <"$TMP/tok2.json")"
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' -X DELETE "$URL/api/v1/tokens/$CAPID")" = 204 ] && pass "DELETE /api/v1/tokens/{id} revokes" || fail "revoke"
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$URL/write" -H "Authorization: Bearer $CAPSECRET" --data-binary 'x v=1')" = 401 ] && pass "a revoked token is 401 at once" || fail "revoked token still accepted"
|
|
|
|
# 4c. Rollups: a 2 y query hits the 1 h tier.
|
|
echo "== (c) rollups and the 2 y graph"
|
|
"$DEVTOOLS" backfill-long --long-url "$LONG" --device "$SW1" --iface "$IFA" --months 25 --days 75 --in-bps 8000000 --out-bps 2000000 >"$TMP/bf1.log" 2>&1 &&
|
|
"$DEVTOOLS" backfill-long --long-url "$LONG" --device "$SW2" --iface "$IFB" --months 25 --days 75 --in-bps 3000000 --out-bps 1000000 >"$TMP/bf2.log" 2>&1 &&
|
|
pass "wrote 25 months of 1 h rollups and 75 days of 5 m rates into the long store for two interfaces" || { cat "$TMP/bf1.log" "$TMP/bf2.log"; die "backfill-long"; }
|
|
"$DEVTOOLS" backfill --hot-url "$HOT" --device "$SW1" --iface "$IFA" --days 1 --pattern flat --mean-mbps 4 >"$TMP/bfhot.log" 2>&1 &&
|
|
pass "wrote a day of raw counters into hot" || { cat "$TMP/bfhot.log"; fail "backfill (hot)"; }
|
|
curl -fsS -X POST "$HOT/internal/force_flush" >/dev/null 2>&1; sleep 1 # make the backfill searchable before the repair reads it
|
|
FROM="$(date -u -d '-2 hours' +%Y-%m-%dT%H:%M:%SZ)"; TO="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
./nmon rollup repair "$FROM" "$TO" --api "$URL" >"$TMP/repair.log" 2>&1 && grep -q 'five-minute windows' "$TMP/repair.log" &&
|
|
pass "nmon rollup repair: $(cat "$TMP/repair.log")" || { cat "$TMP/repair.log"; fail "rollup repair"; }
|
|
T90="$(date -u -d '-90 minutes' +%s)"
|
|
REPAIRED="$(vm_value "$LONG" 'last_over_time(nmon_if_rate_bps_5m{iface="'"$IFA"'",dir="in"}[10m])' "$T90")"
|
|
[ -n "$REPAIRED" ] && near "$REPAIRED" 4000000 0.05 && pass "the repair turned the raw counters into a 5 m rate of $REPAIRED bit/s in the long store" ||
|
|
fail "repaired 5 m rate: '$REPAIRED'"
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$URL/api/v1/rollup/repair" -H 'Content-Type: application/json' \
|
|
-d "{\"from\":\"$(date -u -d '-40 days' +%Y-%m-%dT%H:%M:%SZ)\",\"to\":\"$TO\"}")" = 422 ] && pass "a repair outside hot retention is a 422" || fail "outside-hot repair accepted"
|
|
|
|
TWOY="$(date -u -d '-2 years' +%Y-%m-%dT%H:%M:%SZ)"
|
|
api "/api/v1/graph?iface=$IFA&dir=in&from=$TWOY&to=$TO" >"$TMP/g2y.json" || die "2 y graph request"
|
|
[ "$(py "'1h' in [t['tier'] for t in d.get('tiers',[])]" <"$TMP/g2y.json")" = True ] && pass "the 2 y graph hits the 1 h tier (tiers: $(py "[t['tier'] for t in d['tiers']]" <"$TMP/g2y.json"))" ||
|
|
{ head -c 600 "$TMP/g2y.json"; fail "2 y graph has no 1h tier"; }
|
|
[ "$(py "d.get('stitched_at') is not None" <"$TMP/g2y.json")" = True ] && pass "stitched_at is set: $(py "d['stitched_at']" <"$TMP/g2y.json")" || fail "no stitched_at"
|
|
LONGN="$(py "sum(1 for t,v in d['series']['in'] if v is not None and t < $(date -u -d "$(py "d['stitched_at']" <"$TMP/g2y.json")" +%s))" <"$TMP/g2y.json")"
|
|
[ "$LONGN" -ge 20 ] && pass "the long part has $LONGN data points" || fail "long part has $LONGN points"
|
|
BAD="$(py "sum(1 for t,v in d['series']['in'] if v is not None and t < $(date -u -d "$(py "d['stitched_at']" <"$TMP/g2y.json")" +%s) and abs(v-8e6)>8e4)" <"$TMP/g2y.json")"
|
|
[ "$BAD" = 0 ] && pass "every long-tier point is the backfilled 8 Mbit/s (highest version, no leak)" || fail "$BAD long-tier points differ from 8 Mbit/s"
|
|
|
|
LASTM="$(date -u -d "$(date -u +%Y-%m-01) -1 month" +%Y-%m)"
|
|
api "/api/v1/p95?iface=$IFA&month=$LASTM" >"$TMP/p95.json" || { fail "GET /api/v1/p95"; echo '{}' >"$TMP/p95.json"; }
|
|
P95IN="$(py "d.get('in_bps','')" <"$TMP/p95.json")"
|
|
[ -n "$P95IN" ] && near "$P95IN" 8000000 0.01 && [ -n "$(py "d['source']" <"$TMP/p95.json")" ] &&
|
|
pass "GET /api/v1/p95 for $LASTM: in $P95IN bit/s, source $(py "d['source']" <"$TMP/p95.json"), coverage $(py "d['coverage']" <"$TMP/p95.json")" ||
|
|
{ cat "$TMP/p95.json"; fail "p95 for $LASTM"; }
|
|
api "/api/v1/graph?iface=$IFA&dir=in&month=$LASTM&p95=true" >"$TMP/gm.json" || fail "month graph"
|
|
[ "$(py "d['p95']['month']" <"$TMP/gm.json")" = "$LASTM" ] && near "$(py "d['p95']['in_bps']" <"$TMP/gm.json")" 8000000 0.01 &&
|
|
pass "the interface graph's p95 for $LASTM comes from the monthly figure (source $(py "d['p95']['source']" <"$TMP/gm.json"))" || { head -c 500 "$TMP/gm.json"; fail "graph month p95"; }
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' "$URL/api/v1/p95?iface=$IFA&month=2001-01")" = 422 ] && pass "p95 for a month without data is a 422" || fail "no-data p95"
|
|
|
|
# 4d. Groups.
|
|
echo "== (d) groups"
|
|
GID="$(curl -fsS -X POST "$URL/api/v1/groups" -H 'Content-Type: application/json' -d '{"name":"gate-pair","kind":"custom"}' | py "d['id']")"
|
|
SINCE="$(date -u -d '-60 days' +%Y-%m-%dT%H:%M:%SZ)"
|
|
c1="$(curl -s -o /dev/null -w '%{http_code}' -X POST "$URL/api/v1/groups/$GID/members" -H 'Content-Type: application/json' \
|
|
-d "{\"interface_id\":\"$IFA\",\"direction\":\"toward\",\"effective_from\":\"$SINCE\"}")"
|
|
c2="$(curl -s -o /dev/null -w '%{http_code}' -X POST "$URL/api/v1/groups/$GID/members" -H 'Content-Type: application/json' \
|
|
-d "{\"interface_id\":\"$IFB\",\"direction\":\"away\",\"effective_from\":\"$SINCE\"}")"
|
|
[ "$c1" = 201 ] && [ "$c2" = 201 ] && pass "group $GID has two members (toward, away)" || fail "member add: $c1 $c2"
|
|
GF="$(date -u -d '-30 hours' +%Y-%m-%dT%H:%M:%SZ)"; GT="$(date -u -d '-26 hours' +%Y-%m-%dT%H:%M:%SZ)"
|
|
api "/api/v1/groups/$GID/graph?from=$GF&to=$GT&step=3600&p95=true&month=$LASTM" >"$TMP/gg.json" || fail "group graph"
|
|
GBAD="$(py "sum(1 for k,w in (('in',9e6),('out',5e6)) for t,v in d['series'][k] if v is not None and abs(v-w)>w*0.01)" <"$TMP/gg.json")"
|
|
GN="$(py "len([1 for t,v in d['series']['in'] if v is not None])" <"$TMP/gg.json")"
|
|
[ "$GBAD" = 0 ] && [ "$GN" -ge 3 ] && pass "the group graph is the oriented sum: in 9 Mbit/s (8 + 1), out 5 Mbit/s (2 + 3), $GN points" ||
|
|
{ head -c 700 "$TMP/gg.json"; fail "group graph sum ($GBAD bad points of $GN)"; }
|
|
api "/api/v1/p95?group=$GID&month=$LASTM" >"$TMP/gp.json" || fail "group p95"
|
|
near "$(py "d['in_bps']" <"$TMP/gp.json")" 9000000 0.01 && near "$(py "d['out_bps']" <"$TMP/gp.json")" 5000000 0.01 &&
|
|
pass "group p95 for $LASTM is computed on the summed series: in $(py "d['in_bps']" <"$TMP/gp.json"), out $(py "d['out_bps']" <"$TMP/gp.json") (source $(py "d['source']" <"$TMP/gp.json"))" ||
|
|
{ cat "$TMP/gp.json"; fail "group p95"; }
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$URL/api/v1/groups/$GID/members" -H 'Content-Type: application/json' \
|
|
-d "{\"interface_id\":\"$IFA\",\"direction\":\"toward\",\"effective_from\":\"$SINCE\"}")" = 409 ] && pass "an overlapping membership is a 409" || fail "overlap accepted"
|
|
|
|
# 4e. Checks.
|
|
echo "== (e) checks"
|
|
CODE="$(curl -s -o "$TMP/chk.json" -w '%{http_code}' -X POST "$URL/api/v1/checks" -H 'Content-Type: application/json' \
|
|
-d "{\"kind\":\"http\",\"target\":\"http://[::1]:$SINK_PORT/healthz\",\"interval_s\":10,\"name\":\"gate-local-http\"}")"
|
|
[ "$CODE" = 201 ] && pass "created an HTTP check against the local server" || { cat "$TMP/chk.json"; die "create check: HTTP $CODE"; }
|
|
[ "$(curl -s -o /dev/null -w '%{http_code}' -X POST "$URL/api/v1/checks" -H 'Content-Type: application/json' -d '{"kind":"http","target":"ftp://x","name":"bad"}')" = 422 ] &&
|
|
pass "an invalid check is a 422" || fail "invalid check accepted"
|
|
CHKDEV="$(py "d['device']['slug']" <"$TMP/chk.json")"
|
|
check_up() { [ "$(api /api/v1/checks | py "next(c['last']['status'] for c in d['items'] if c['device']['slug']=='$CHKDEV')" 2>/dev/null)" = up ]; }
|
|
wait_for 60 "the check to read up" check_up && pass "the check ran through the outbound allow-list and reads up" || { api /api/v1/checks; tail -n 5 "$TMP/nmon.log"; fail "check not up"; }
|
|
cup() { [ "$(vm_value "$HOT" 'nmon_check_up{device="'"$CHKDEV"'"}')" = 1 ]; }
|
|
wait_for 40 "nmon_check_up in hot" cup && pass "nmon_check_up{device,check} = 1 is in the hot store" || fail "no nmon_check_up sample"
|
|
[ "$(api "/api/v1/devices/$CHKDEV" | py "len(d['checks'])")" = 1 ] && pass "the check_only device page lists its check" || fail "device checks"
|
|
chk_reach() { [ "$(api "/api/v1/devices/$CHKDEV" | py "d['reachability']")" = up ]; }
|
|
wait_for 30 "the check_only device to read up" chk_reach && pass "the check_only device reads up from its check" || fail "check_only reachability"
|
|
BLOCKED="$(curl -s -X POST "$URL/api/v1/checks" -H 'Content-Type: application/json' -d '{"kind":"http","target":"http://169.254.169.254/","interval_s":10,"name":"gate-linklocal"}' | py "d['id']")"
|
|
blocked_down() { [ "$(api /api/v1/checks | py "next(c['last']['status'] for c in d['items'] if c['id']=='$BLOCKED')" 2>/dev/null)" = down ]; }
|
|
wait_for 40 "the link-local check to fail" blocked_down && pass "a check to a denied address fails (outbound policy applies)" || fail "link-local check not denied"
|
|
|
|
# 5. OpenAPI and the web client.
|
|
echo "== openapi"
|
|
curl -fsS "$URL/openapi.json" >"$TMP/openapi.live.json" || die "GET /openapi.json"
|
|
OPENAPI_BASE_TAG=m3 OPENAPI_NEW="$TMP/openapi.live.json" ./scripts/openapi-check.sh && pass "openapi-check against m3: additive only" || fail "openapi-check"
|
|
if ./nmon openapi >"$TMP/openapi.cli.json" 2>/dev/null; then
|
|
cmp -s "$TMP/openapi.cli.json" openapi.json && pass "openapi.json matches nmon openapi" || fail "openapi.json is stale: run ./nmon openapi > openapi.json"
|
|
fi
|
|
echo "== contract drift"
|
|
cp web/src/api/schema.d.ts "$TMP/schema.before.d.ts"
|
|
if (cd web && npm run gen:api -- "$URL/openapi.json" >"$TMP/genapi.log" 2>&1 && npm run build >"$TMP/webbuild.log" 2>&1); then
|
|
pass "web client generated from the live spec type-checks and builds"
|
|
else
|
|
tail -n 30 "$TMP/genapi.log" "$TMP/webbuild.log" 2>/dev/null; fail "web build against the live spec"
|
|
fi
|
|
cmp -s web/src/api/schema.d.ts "$TMP/schema.before.d.ts" && pass "web/src/api/schema.d.ts is current" ||
|
|
echo "NOTE: web/src/api/schema.d.ts changed after regeneration from the live spec; review and commit it"
|
|
(cd web && npm run lint >"$TMP/weblint.log" 2>&1) && pass "web lint after regeneration" || { tail -n 20 "$TMP/weblint.log"; fail "web lint"; }
|
|
echo "== playwright (mock)"
|
|
if (cd web && npm run e2e) >"$TMP/e2e-mock.log" 2>&1; then pass "mock UI specs (incl. docs/screenshots/M5-mock)"; else tail -n 40 "$TMP/e2e-mock.log"; fail "mock UI specs"; fi
|
|
|
|
# 6. Live screenshots.
|
|
echo "== screenshots (live)"
|
|
rm -rf "$SHOTS"; mkdir -p "$SHOTS"
|
|
VMGUEST="$GUEST"
|
|
export NMON_URL="$URL" GATE_GROUP_ID="$GID" GATE_VM_SLUG="$VMGUEST" GATE_NODE_SLUG="$PVE1" GATE_CHECK_DEVICE_SLUG="$CHKDEV" GATE_IFACE_ID="$IFA"
|
|
if (cd web && NMON_SHOT_DIR="$SHOTS" npm run e2e:live -- screenshots-m5.spec.ts) >"$TMP/shots.log" 2>&1; then
|
|
N="$(ls "$SHOTS"/*.png 2>/dev/null | wc -l)"
|
|
[ "$N" -ge 5 ] && pass "$N screenshots in $SHOTS" || fail "only $N screenshots in $SHOTS"
|
|
ls "$SHOTS"
|
|
else
|
|
tail -n 40 "$TMP/shots.log"; fail "screenshots"
|
|
fi
|
|
|
|
if [ "$FAILS" -eq 0 ]; then echo "GATE M5: PASS"; else echo "GATE M5: FAIL ($FAILS)"; exit 1; fi
|