fix: recover monitoring charts after missing samples

This commit is contained in:
2026-11-01 22:54:01 +08:00
parent 0820db0a5d
commit 8ec5e4e324
7 changed files with 170 additions and 33 deletions
+33 -10
View File
@@ -16,15 +16,15 @@ type metricsHistory struct {
Samples []map[string]float64 `json:"samples"`
}
type metricStats struct {
Mean float64 `json:"mean"`
Latest float64 `json:"latest"`
Max float64 `json:"max"`
Mean float64 `json:"mean"`
Latest *float64 `json:"latest"`
Max float64 `json:"max"`
}
type metricSeries struct {
ID string `json:"id"`
LabelKey string `json:"labelKey"`
Color string `json:"color"`
Values []float64 `json:"values"`
Values []*float64 `json:"values"`
Stats metricStats `json:"stats"`
}
type metricPanel struct {
@@ -91,33 +91,53 @@ func (a *App) monitoring(w http.ResponseWriter, r *http.Request) {
timestamps = append(timestamps, int64(samples[index]["time"]))
}
makeSeries := func(id, label, color string) metricSeries {
series := metricSeries{ID: id, LabelKey: "monitor.series." + label, Color: "var(--" + color + ")", Values: []float64{}}
series := metricSeries{ID: id, LabelKey: "monitor.series." + label, Color: "var(--" + color + ")", Values: []*float64{}}
count := 0
for _, sample := range samples {
if value, ok := sample[id]; ok {
series.Stats.Mean += value
series.Stats.Max = math.Max(value, series.Stats.Max)
series.Stats.Latest = value
count++
}
}
if count > 0 {
series.Stats.Mean /= float64(count)
}
if len(samples) > 0 {
if latest, ok := samples[len(samples)-1][id]; ok {
series.Stats.Latest = &latest
}
}
previousIndex := -1
for _, index := range indices {
series.Values = append(series.Values, samples[index][id])
var point *float64
complete := true
// Preserve gaps even when downsampling skips the original missing point.
for i := previousIndex + 1; i <= index; i++ {
if _, ok := samples[i][id]; !ok {
complete = false
break
}
}
if complete {
value := samples[index][id]
point = &value
}
series.Values = append(series.Values, point)
previousIndex = index
}
return series
}
panel := func(id, unit string, maximum float64, stacked bool, series ...metricSeries) metricPanel {
available := len(samples) > 1
completePoints := 0
peak := float64(0)
for _, sample := range samples {
sum := float64(0)
complete := true
for _, s := range series {
value, ok := sample[s.ID]
if !ok {
available = false
complete = false
}
if stacked {
sum += value
@@ -126,6 +146,9 @@ func (a *App) monitoring(w http.ResponseWriter, r *http.Request) {
}
}
peak = math.Max(peak, sum)
if complete {
completePoints++
}
}
if maximum == 0 {
maximum = niceMaximum(peak)
@@ -133,7 +156,7 @@ func (a *App) monitoring(w http.ResponseWriter, r *http.Request) {
if maximum <= 0 {
maximum = 1
}
return metricPanel{id, "monitor.charts." + id, unit, maximum, stacked, available, series}
return metricPanel{id, "monitor.charts." + id, unit, maximum, stacked, completePoints > 1, series}
}
panels := []metricPanel{
panel("ipv4", "gigabits", 0, true, makeSeries("https4", "https", "blue"), makeSeries("rsync4", "rsync", "violet")),
+76
View File
@@ -0,0 +1,76 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
"time"
)
func TestMonitoringRecoversAfterMissingSamples(t *testing.T) {
c := fixture(t)
path := filepath.Join(t.TempDir(), "history.json")
c.app.cfg.Metrics = path
now := time.Now()
history := metricsHistory{UpdatedAt: now.UnixMilli(), MemoryTotal: 32}
for i := 0; i < 500; i++ {
sample := map[string]float64{"time": float64(now.Add(time.Duration(i-500) * time.Second).UnixMilli())}
for _, key := range []string{"https4", "rsync4", "readOps", "writeOps", "readBytes", "writeBytes"} {
sample[key] = 4
}
history.Samples = append(history.Samples, sample)
}
// A collector startup or counter reset must not disable an entire history window.
for _, key := range []string{"https4", "rsync4", "readOps", "writeOps", "readBytes", "writeBytes"} {
delete(history.Samples[0], key)
delete(history.Samples[1], key) // Also exercise a gap skipped by downsampling.
delete(history.Samples[100], key)
}
delete(history.Samples[499], "https4")
history.Samples[499]["rsync4"] = 0 // A measured zero is different from missing data.
raw, _ := json.Marshal(history)
if err := os.WriteFile(path, raw, 0644); err != nil {
t.Fatal(err)
}
for _, hours := range []int{1, 6, 24} {
t.Run(fmt.Sprint(hours), func(t *testing.T) {
response := c.call("GET", fmt.Sprintf("/api/v1/monitoring?hours=%d", hours), nil, false)
status(t, response, 200)
var result struct {
Panels []metricPanel `json:"panels"`
Timestamps []int64 `json:"timestamps"`
}
if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil {
t.Fatal(err)
}
for _, panel := range result.Panels {
wanted := panel.ID == "ipv4" || panel.ID == "iops" || panel.ID == "throughput"
if panel.Available != wanted {
t.Fatalf("panel %s available=%t, want %t", panel.ID, panel.Available, wanted)
}
for _, series := range panel.Series {
if len(series.Values) != len(result.Timestamps) {
t.Fatal("series no longer aligns with shared timestamps")
}
if wanted {
if series.Values[0] != nil || series.Values[1] != nil {
t.Fatal("startup gap lost during downsampling")
}
if series.Stats.Max != 4 || series.Stats.Mean <= 3.9 {
t.Fatal("missing samples polluted statistics")
}
}
}
}
v4 := result.Panels[0].Series
if v4[0].Stats.Latest != nil || v4[0].Values[len(v4[0].Values)-1] != nil {
t.Fatal("missing latest measurement presented as zero or a stale value")
}
if latest := v4[1].Stats.Latest; latest == nil || *latest != 0 {
t.Fatal("valid zero measurement was dropped")
}
})
}
}