fix: recover monitoring charts after missing samples

This commit is contained in:
2026-11-01 22:54:01 +08:00
parent 0820db0a5d
commit 8ec5e4e324
7 changed files with 170 additions and 33 deletions
+31 -8
View File
@@ -17,14 +17,14 @@ type metricsHistory struct {
} }
type metricStats struct { type metricStats struct {
Mean float64 `json:"mean"` Mean float64 `json:"mean"`
Latest float64 `json:"latest"` Latest *float64 `json:"latest"`
Max float64 `json:"max"` Max float64 `json:"max"`
} }
type metricSeries struct { type metricSeries struct {
ID string `json:"id"` ID string `json:"id"`
LabelKey string `json:"labelKey"` LabelKey string `json:"labelKey"`
Color string `json:"color"` Color string `json:"color"`
Values []float64 `json:"values"` Values []*float64 `json:"values"`
Stats metricStats `json:"stats"` Stats metricStats `json:"stats"`
} }
type metricPanel struct { type metricPanel struct {
@@ -91,33 +91,53 @@ func (a *App) monitoring(w http.ResponseWriter, r *http.Request) {
timestamps = append(timestamps, int64(samples[index]["time"])) timestamps = append(timestamps, int64(samples[index]["time"]))
} }
makeSeries := func(id, label, color string) metricSeries { makeSeries := func(id, label, color string) metricSeries {
series := metricSeries{ID: id, LabelKey: "monitor.series." + label, Color: "var(--" + color + ")", Values: []float64{}} series := metricSeries{ID: id, LabelKey: "monitor.series." + label, Color: "var(--" + color + ")", Values: []*float64{}}
count := 0 count := 0
for _, sample := range samples { for _, sample := range samples {
if value, ok := sample[id]; ok { if value, ok := sample[id]; ok {
series.Stats.Mean += value series.Stats.Mean += value
series.Stats.Max = math.Max(value, series.Stats.Max) series.Stats.Max = math.Max(value, series.Stats.Max)
series.Stats.Latest = value
count++ count++
} }
} }
if count > 0 { if count > 0 {
series.Stats.Mean /= float64(count) series.Stats.Mean /= float64(count)
} }
if len(samples) > 0 {
if latest, ok := samples[len(samples)-1][id]; ok {
series.Stats.Latest = &latest
}
}
previousIndex := -1
for _, index := range indices { for _, index := range indices {
series.Values = append(series.Values, samples[index][id]) var point *float64
complete := true
// Preserve gaps even when downsampling skips the original missing point.
for i := previousIndex + 1; i <= index; i++ {
if _, ok := samples[i][id]; !ok {
complete = false
break
}
}
if complete {
value := samples[index][id]
point = &value
}
series.Values = append(series.Values, point)
previousIndex = index
} }
return series return series
} }
panel := func(id, unit string, maximum float64, stacked bool, series ...metricSeries) metricPanel { panel := func(id, unit string, maximum float64, stacked bool, series ...metricSeries) metricPanel {
available := len(samples) > 1 completePoints := 0
peak := float64(0) peak := float64(0)
for _, sample := range samples { for _, sample := range samples {
sum := float64(0) sum := float64(0)
complete := true
for _, s := range series { for _, s := range series {
value, ok := sample[s.ID] value, ok := sample[s.ID]
if !ok { if !ok {
available = false complete = false
} }
if stacked { if stacked {
sum += value sum += value
@@ -126,6 +146,9 @@ func (a *App) monitoring(w http.ResponseWriter, r *http.Request) {
} }
} }
peak = math.Max(peak, sum) peak = math.Max(peak, sum)
if complete {
completePoints++
}
} }
if maximum == 0 { if maximum == 0 {
maximum = niceMaximum(peak) maximum = niceMaximum(peak)
@@ -133,7 +156,7 @@ func (a *App) monitoring(w http.ResponseWriter, r *http.Request) {
if maximum <= 0 { if maximum <= 0 {
maximum = 1 maximum = 1
} }
return metricPanel{id, "monitor.charts." + id, unit, maximum, stacked, available, series} return metricPanel{id, "monitor.charts." + id, unit, maximum, stacked, completePoints > 1, series}
} }
panels := []metricPanel{ panels := []metricPanel{
panel("ipv4", "gigabits", 0, true, makeSeries("https4", "https", "blue"), makeSeries("rsync4", "rsync", "violet")), panel("ipv4", "gigabits", 0, true, makeSeries("https4", "https", "blue"), makeSeries("rsync4", "rsync", "violet")),
+76
View File
@@ -0,0 +1,76 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
"time"
)
func TestMonitoringRecoversAfterMissingSamples(t *testing.T) {
c := fixture(t)
path := filepath.Join(t.TempDir(), "history.json")
c.app.cfg.Metrics = path
now := time.Now()
history := metricsHistory{UpdatedAt: now.UnixMilli(), MemoryTotal: 32}
for i := 0; i < 500; i++ {
sample := map[string]float64{"time": float64(now.Add(time.Duration(i-500) * time.Second).UnixMilli())}
for _, key := range []string{"https4", "rsync4", "readOps", "writeOps", "readBytes", "writeBytes"} {
sample[key] = 4
}
history.Samples = append(history.Samples, sample)
}
// A collector startup or counter reset must not disable an entire history window.
for _, key := range []string{"https4", "rsync4", "readOps", "writeOps", "readBytes", "writeBytes"} {
delete(history.Samples[0], key)
delete(history.Samples[1], key) // Also exercise a gap skipped by downsampling.
delete(history.Samples[100], key)
}
delete(history.Samples[499], "https4")
history.Samples[499]["rsync4"] = 0 // A measured zero is different from missing data.
raw, _ := json.Marshal(history)
if err := os.WriteFile(path, raw, 0644); err != nil {
t.Fatal(err)
}
for _, hours := range []int{1, 6, 24} {
t.Run(fmt.Sprint(hours), func(t *testing.T) {
response := c.call("GET", fmt.Sprintf("/api/v1/monitoring?hours=%d", hours), nil, false)
status(t, response, 200)
var result struct {
Panels []metricPanel `json:"panels"`
Timestamps []int64 `json:"timestamps"`
}
if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil {
t.Fatal(err)
}
for _, panel := range result.Panels {
wanted := panel.ID == "ipv4" || panel.ID == "iops" || panel.ID == "throughput"
if panel.Available != wanted {
t.Fatalf("panel %s available=%t, want %t", panel.ID, panel.Available, wanted)
}
for _, series := range panel.Series {
if len(series.Values) != len(result.Timestamps) {
t.Fatal("series no longer aligns with shared timestamps")
}
if wanted {
if series.Values[0] != nil || series.Values[1] != nil {
t.Fatal("startup gap lost during downsampling")
}
if series.Stats.Max != 4 || series.Stats.Mean <= 3.9 {
t.Fatal("missing samples polluted statistics")
}
}
}
}
v4 := result.Panels[0].Series
if v4[0].Stats.Latest != nil || v4[0].Values[len(v4[0].Values)-1] != nil {
t.Fatal("missing latest measurement presented as zero or a stale value")
}
if latest := v4[1].Stats.Latest; latest == nil || *latest != 0 {
t.Fatal("valid zero measurement was dropped")
}
})
}
}
+14 -3
View File
@@ -1,5 +1,5 @@
import assert from 'node:assert/strict' import assert from 'node:assert/strict'
import { createMonitoringSnapshot, demoDisks, type MonitoringRange } from '../src/monitoring.ts' import { createMonitoringSnapshot, demoDisks, metricPaths, metricStatistics, type MetricSeries, type MonitoringRange } from '../src/monitoring.ts'
const end = Date.parse('2026-11-01T16:00:00+08:00') const end = Date.parse('2026-11-01T16:00:00+08:00')
for (const hours of [1, 6, 24] as MonitoringRange[]) { for (const hours of [1, 6, 24] as MonitoringRange[]) {
@@ -12,7 +12,7 @@ for (const hours of [1, 6, 24] as MonitoringRange[]) {
for (const panel of snapshot.panels) { for (const panel of snapshot.panels) {
for (const series of panel.series) { for (const series of panel.series) {
assert.equal(series.values.length, snapshot.timestamps.length) assert.equal(series.values.length, snapshot.timestamps.length)
assert.ok(series.values.every(value => Number.isFinite(value) && value >= 0 && value <= panel.maximum)) assert.ok(series.values.every(value => value !== null && Number.isFinite(value) && value >= 0 && value <= panel.maximum))
} }
for (let index = 0; index < snapshot.timestamps.length; index++) { for (let index = 0; index < snapshot.timestamps.length; index++) {
if (index) assert.ok(snapshot.timestamps[index]! > snapshot.timestamps[index - 1]!) if (index) assert.ok(snapshot.timestamps[index]! > snapshot.timestamps[index - 1]!)
@@ -23,4 +23,15 @@ for (const hours of [1, 6, 24] as MonitoringRange[]) {
} }
} }
for (const disk of demoDisks) assert.ok(disk.used >= 0 && disk.used <= disk.total) for (const disk of demoDisks) assert.ok(disk.used >= 0 && disk.used <= disk.total)
console.log('Monitoring check passed: deterministic windows, timestamps, valid samples, stack limits, CPU/memory totals and disk capacities.') const series = (id: string, values: MetricSeries['values']): MetricSeries => ({ id, values, labelKey: id, color: 'blue' })
const lower = series('lower', [0, 2, null, 3, 4])
const upper = series('upper', [1, 1, 1, 1, 1])
const coordinate = (index: number, value: number) => `${index},${value}`
const paths = metricPaths([lower, upper], 5, true, coordinate)
assert.equal(paths[0]!.line, 'M0,0 L1,2 M3,3 L4,4', 'Missing data must break a line, preserving measured zero')
assert.equal(paths[0]!.area, 'M0,0 L1,2 L1,0 L0,0 Z M3,3 L4,4 L4,0 L3,0 Z', 'Fills must not bridge gaps')
assert.deepEqual(paths[1]!.plottedValues, [1, 3, null, 4, 5], 'Unknown stacked baselines must remain gaps')
assert.deepEqual(metricPaths([upper], 5, true, coordinate)[0]!.plottedValues, upper.values, 'Hiding a lower series must rebuild the stack')
assert.deepEqual(metricStatistics([null, 0, 4, null]), { mean: 2, latest: null, max: 4 }, 'Missing samples must not become zero or stale latest values')
assert.deepEqual(metricStatistics([null, null]), { mean: 0, latest: null, max: 0 })
console.log('Monitoring check passed: deterministic windows, statistics, missing samples, line/fill gaps, stack limits and disk capacities.')
+6 -17
View File
@@ -1,7 +1,7 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, onBeforeUnmount, onMounted, ref } from 'vue' import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
import { useI18n } from 'vue-i18n' import { useI18n } from 'vue-i18n'
import type { MonitoringPanel } from '../monitoring' import { metricPaths, metricStatistics, type MonitoringPanel } from '../monitoring'
const props = defineProps<{ panel: MonitoringPanel; timestamps: number[] }>() const props = defineProps<{ panel: MonitoringPanel; timestamps: number[] }>()
const { t, d, n } = useI18n({ useScope: 'global' }) const { t, d, n } = useI18n({ useScope: 'global' })
@@ -19,7 +19,8 @@ const plotWidth = computed(() => Math.max(1, width.value - left - right))
const plotHeight = computed(() => height.value - top - bottom) const plotHeight = computed(() => height.value - top - bottom)
const x = (index: number) => left + index / Math.max(1, props.timestamps.length - 1) * plotWidth.value const x = (index: number) => left + index / Math.max(1, props.timestamps.length - 1) * plotWidth.value
const y = (value: number) => top + (1 - value / props.panel.maximum) * plotHeight.value const y = (value: number) => top + (1 - value / props.panel.maximum) * plotHeight.value
function format(value: number) { function format(value: number | null | undefined) {
if (value == null || !Number.isFinite(value)) return t('common.emptyValue')
const unit = props.panel.unit const unit = props.panel.unit
if (value > 0 && value < 1 && (unit === 'gigabits' || unit === 'gigabytes')) { if (value > 0 && value < 1 && (unit === 'gigabits' || unit === 'gigabytes')) {
return t(`monitor.units.${unit === 'gigabits' ? 'megabits' : 'megabytes'}`, { value: n(value * 1000, 'size') }) return t(`monitor.units.${unit === 'gigabits' ? 'megabits' : 'megabytes'}`, { value: n(value * 1000, 'size') })
@@ -32,22 +33,10 @@ const timeTicks = computed(() => {
return Array.from({ length: count }, (_, i) => Math.round((props.timestamps.length - 1) * i / (count - 1))) return Array.from({ length: count }, (_, i) => Math.round((props.timestamps.length - 1) * i / (count - 1)))
}) })
const visible = computed(() => props.panel.series.filter(series => !hidden.value.includes(series.id))) const visible = computed(() => props.panel.series.filter(series => !hidden.value.includes(series.id)))
const paths = computed(() => { const paths = computed(() => metricPaths(visible.value, props.timestamps.length, props.panel.stacked, (index, value) => `${x(index).toFixed(2)},${y(value).toFixed(2)}`))
let baseline = props.timestamps.map(() => 0)
return visible.value.map(series => {
const base = [...baseline]
const values = series.values.map((value, i) => value + (props.panel.stacked ? base[i]! : 0))
if (props.panel.stacked) baseline = values
const line = values.map((value, i) => `${i ? 'L' : 'M'}${x(i).toFixed(2)},${y(value).toFixed(2)}`).join(' ')
const floor = base.map((value, i) => `L${x(i).toFixed(2)},${y(props.panel.stacked ? value : 0).toFixed(2)}`).reverse().join(' ')
return { ...series, line, area: `${line} ${floor} Z`, plottedValues: values }
})
})
const stats = computed(() => props.panel.series.map(series => ({ const stats = computed(() => props.panel.series.map(series => ({
...series, ...series,
mean: series.values.reduce((sum, value) => sum + value, 0) / Math.max(1, series.values.length), ...metricStatistics(series.values),
latest: series.values.at(-1) ?? 0,
max: Math.max(0, ...series.values),
...series.stats, ...series.stats,
}))) })))
const available = computed(() => props.panel.available !== false && props.timestamps.length > 1) const available = computed(() => props.panel.available !== false && props.timestamps.length > 1)
@@ -109,7 +98,7 @@ onBeforeUnmount(() => observer?.disconnect())
</g> </g>
<g v-if="sampleIndex !== null" class="chart-cursor"> <g v-if="sampleIndex !== null" class="chart-cursor">
<line :x1="x(sampleIndex)" :x2="x(sampleIndex)" :y1="top" :y2="height - bottom" /> <line :x1="x(sampleIndex)" :x2="x(sampleIndex)" :y1="top" :y2="height - bottom" />
<circle v-for="series in paths" :key="series.id" :cx="x(sampleIndex)" :cy="y(series.plottedValues[sampleIndex]!)" r="3" :fill="series.color" /> <template v-for="series in paths" :key="series.id"><circle v-if="series.plottedValues[sampleIndex] != null" :cx="x(sampleIndex)" :cy="y(series.plottedValues[sampleIndex]!)" r="3" :fill="series.color" /></template>
</g> </g>
</svg> </svg>
<div v-if="available && sampleIndex !== null" class="chart-tooltip" :style="{ left: tooltipLeft }" aria-hidden="true"> <div v-if="available && sampleIndex !== null" class="chart-tooltip" :style="{ left: tooltipLeft }" aria-hidden="true">
+1 -1
View File
@@ -1,6 +1,6 @@
export default { export default {
speedCollecting: 'Sampling', speedCollecting: 'Sampling',
speedDescription: 'TCP receive rate across this mirror’s parallel upstream connections, sampled about every 2 seconds.', speedDescription: 'TCP receive rate across this mirror’s parallel upstream connections, sampled about every 2 seconds. Scanning, verification or disk waits may produce zero while the sync job is still running.',
speedUnits: { bytes: '{value} B/s', kilobytes: '{value} KB/s', megabytes: '{value} MB/s', gigabytes: '{value} GB/s' }, speedUnits: { bytes: '{value} B/s', kilobytes: '{value} KB/s', megabytes: '{value} MB/s', gigabytes: '{value} GB/s' },
live: 'Live monitoring', live: 'Live monitoring',
stale: 'Metrics collection interrupted', stale: 'Metrics collection interrupted',
+1 -1
View File
@@ -1,6 +1,6 @@
export default { export default {
speedCollecting: '采集中', speedCollecting: '采集中',
speedDescription: '约 2 秒采样窗口内,此仓库全部并行上游连接的 TCP 接收速率。', speedDescription: '约 2 秒采样窗口内,此仓库全部并行上游连接的 TCP 接收速率。扫描、校验或等待磁盘时可能为 0,并不表示同步已停止。',
speedUnits: { bytes: '{value} B/s', kilobytes: '{value} KB/s', megabytes: '{value} MB/s', gigabytes: '{value} GB/s' }, speedUnits: { bytes: '{value} B/s', kilobytes: '{value} KB/s', megabytes: '{value} MB/s', gigabytes: '{value} GB/s' },
live: '实时监控', live: '实时监控',
stale: '数据采集暂时中断', stale: '数据采集暂时中断',
+39 -1
View File
@@ -1,6 +1,7 @@
export type MonitoringRange = 1 | 6 | 24 export type MonitoringRange = 1 | 6 | 24
export type MetricUnit = 'gigabits' | 'gigabytes' | 'operations' | 'percent' | 'memory' export type MetricUnit = 'gigabits' | 'gigabytes' | 'operations' | 'percent' | 'memory'
export interface MetricSeries { id: string; labelKey: string; color: string; values: number[]; stats?: { mean: number; latest: number; max: number } } export type MetricValue = number | null
export interface MetricSeries { id: string; labelKey: string; color: string; values: MetricValue[]; stats?: { mean: number; latest: MetricValue; max: number } }
export interface MonitoringPanel { export interface MonitoringPanel {
id: string id: string
titleKey: string titleKey: string
@@ -13,6 +14,43 @@ export interface MonitoringPanel {
export interface MonitoringSnapshot { timestamps: number[]; panels: MonitoringPanel[] } export interface MonitoringSnapshot { timestamps: number[]; panels: MonitoringPanel[] }
export interface LiveMonitoringSnapshot extends MonitoringSnapshot { disks: { name: string; used: number; total: number }[]; updatedAt: number; stale: boolean } export interface LiveMonitoringSnapshot extends MonitoringSnapshot { disks: { name: string; used: number; total: number }[]; updatedAt: number; stale: boolean }
export function metricStatistics(values: MetricValue[]) {
const measured = values.filter((value): value is number => value !== null && Number.isFinite(value))
const latest = values.at(-1)
return {
mean: measured.reduce((sum, value) => sum + value, 0) / Math.max(1, measured.length),
latest: latest != null && Number.isFinite(latest) ? latest : null,
max: Math.max(0, ...measured),
}
}
// Missing measurements break both lines and fills; they are never plotted as zero.
export function metricPaths(series: MetricSeries[], count: number, stacked: boolean, coordinate: (index: number, value: number) => string) {
let baseline: MetricValue[] = Array.from({ length: count }, () => 0)
return series.map(item => {
const base = baseline
const values = Array.from({ length: count }, (_, index) => {
const value = item.values[index]
if (value == null || !Number.isFinite(value) || (stacked && base[index] == null)) return null
return value + (stacked ? base[index]! : 0)
})
if (stacked) baseline = values
const segments: number[][] = []
let segment: number[] = []
for (let index = 0; index <= count; index++) {
if (index < count && values[index] != null) segment.push(index)
else if (segment.length) { segments.push(segment); segment = [] }
}
const lineFor = (indices: number[]) => indices.map((index, position) => `${position ? 'L' : 'M'}${coordinate(index, values[index]!)}`).join(' ')
return {
...item,
line: segments.map(lineFor).join(' '),
area: segments.map(indices => `${lineFor(indices)} ${[...indices].reverse().map(index => `L${coordinate(index, stacked ? base[index]! : 0)}`).join(' ')} Z`).join(' '),
plottedValues: values,
}
})
}
export const demoDisks = [ export const demoDisks = [
{ name: '/srv/mirrors', used: 43.2, total: 64 }, { name: '/srv/mirrors', used: 43.2, total: 64 },
{ name: '/srv/archive', used: 12.8, total: 32 }, { name: '/srv/archive', used: 12.8, total: 32 },