From 4c3b7ead4e8f6694487c6a2c588f8fa3f9cb67ef Mon Sep 17 00:00:00 2001 From: yaosanqi137 Date: Mon, 2 Nov 2026 14:13:26 +0800 Subject: [PATCH] fix: account for HTTP/S traffic and separate network curves --- backend/monitoring.go | 4 +- backend/monitoring_test.go | 40 ++++- backend/security_test.go | 2 +- backend/tools/collect_metrics.py | 2 +- backend/tools/protocol_counters.py | 197 ++++++++++++++++++++++++ backend/tools/test_protocol_counters.py | 78 ++++++++++ scripts/check-monitoring.ts | 14 ++ src/locales/en-US/monitor.ts | 2 +- src/locales/zh-CN/monitor.ts | 2 +- src/monitoring.ts | 8 +- 10 files changed, 336 insertions(+), 13 deletions(-) create mode 100644 backend/tools/protocol_counters.py create mode 100644 backend/tools/test_protocol_counters.py diff --git a/backend/monitoring.go b/backend/monitoring.go index 674af7f..09b4530 100644 --- a/backend/monitoring.go +++ b/backend/monitoring.go @@ -159,8 +159,8 @@ func (a *App) monitoring(w http.ResponseWriter, r *http.Request) { return metricPanel{id, "monitor.charts." + id, unit, maximum, stacked, completePoints > 1, series} } panels := []metricPanel{ - panel("ipv4", "gigabits", 0, true, makeSeries("https4", "https", "blue"), makeSeries("rsync4", "rsync", "violet")), - panel("ipv6", "gigabits", 0, true, makeSeries("https6", "https", "blue"), makeSeries("rsync6", "rsync", "violet")), + panel("ipv4", "gigabits", 0, false, makeSeries("http4", "http", "blue"), makeSeries("rsync4", "rsync", "violet")), + panel("ipv6", "gigabits", 0, false, makeSeries("http6", "http", "blue"), makeSeries("rsync6", "rsync", "violet")), panel("iops", "operations", 0, false, makeSeries("readOps", "read", "green"), makeSeries("writeOps", "write", "amber")), panel("throughput", "gigabytes", 0, false, makeSeries("readBytes", "read", "blue"), makeSeries("writeBytes", "write", "violet")), panel("cpu", "percent", 100, true, makeSeries("user", "user", "blue"), makeSeries("system", "system", "green"), makeSeries("wait", "wait", "amber"), makeSeries("idle", "idle", "chart-muted")), diff --git a/backend/monitoring_test.go b/backend/monitoring_test.go index 22a8d44..75477f7 100644 --- a/backend/monitoring_test.go +++ b/backend/monitoring_test.go @@ -17,18 +17,18 @@ func TestMonitoringRecoversAfterMissingSamples(t *testing.T) { history := metricsHistory{UpdatedAt: now.UnixMilli(), MemoryTotal: 32} for i := 0; i < 500; i++ { sample := map[string]float64{"time": float64(now.Add(time.Duration(i-500) * time.Second).UnixMilli())} - for _, key := range []string{"https4", "rsync4", "readOps", "writeOps", "readBytes", "writeBytes"} { + for _, key := range []string{"http4", "rsync4", "readOps", "writeOps", "readBytes", "writeBytes"} { sample[key] = 4 } history.Samples = append(history.Samples, sample) } // A collector startup or counter reset must not disable an entire history window. - for _, key := range []string{"https4", "rsync4", "readOps", "writeOps", "readBytes", "writeBytes"} { + for _, key := range []string{"http4", "rsync4", "readOps", "writeOps", "readBytes", "writeBytes"} { delete(history.Samples[0], key) delete(history.Samples[1], key) // Also exercise a gap skipped by downsampling. delete(history.Samples[100], key) } - delete(history.Samples[499], "https4") + delete(history.Samples[499], "http4") history.Samples[499]["rsync4"] = 0 // A measured zero is different from missing data. raw, _ := json.Marshal(history) if err := os.WriteFile(path, raw, 0644); err != nil { @@ -74,3 +74,37 @@ func TestMonitoringRecoversAfterMissingSamples(t *testing.T) { }) } } + +func TestMonitoringDoesNotRelabelHTTPSOnlyHistory(t *testing.T) { + c := fixture(t) + path := filepath.Join(t.TempDir(), "history.json") + c.app.cfg.Metrics = path + now := time.Now() + history := metricsHistory{UpdatedAt: now.UnixMilli(), Samples: []map[string]float64{ + {"time": float64(now.Add(-time.Minute).UnixMilli()), "https4": 20, "rsync4": 3}, + {"time": float64(now.Add(-20 * time.Second).UnixMilli()), "http4": 3, "rsync4": 3}, + {"time": float64(now.Add(-10 * time.Second).UnixMilli()), "http4": 3, "rsync4": 3}, + }} + raw, _ := json.Marshal(history) + if err := os.WriteFile(path, raw, 0644); err != nil { + t.Fatal(err) + } + response := c.call("GET", "/api/v1/monitoring?hours=1", nil, false) + status(t, response, 200) + var result struct{ Panels []metricPanel } + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + series := result.Panels[0].Series[0] + if !result.Panels[0].Available || series.ID != "http4" || series.LabelKey != "monitor.series.http" || series.Values[0] != nil || series.Stats.Mean != 3 || series.Stats.Max != 3 { + t.Fatal("HTTPS-only history was incorrectly presented as complete HTTP/S traffic") + } + for _, panel := range result.Panels[:2] { + if panel.Stacked { + t.Fatalf("%s must plot each protocol's actual rate independently", panel.ID) + } + } + if result.Panels[0].Maximum != 5 { + t.Fatal("network axis must use the largest individual rate, not their sum") + } +} diff --git a/backend/security_test.go b/backend/security_test.go index 8e6d2bc..186a10b 100644 --- a/backend/security_test.go +++ b/backend/security_test.go @@ -50,7 +50,7 @@ func TestMonitoringUsesRawStatistics(t *testing.T) { if index == 1 { value = 200 } - history.Samples = append(history.Samples, map[string]float64{"time": float64(time.Now().Add(time.Duration(index-500) * time.Second).UnixMilli()), "https4": value, "rsync4": 0}) + history.Samples = append(history.Samples, map[string]float64{"time": float64(time.Now().Add(time.Duration(index-500) * time.Second).UnixMilli()), "http4": value, "rsync4": 0}) } raw, _ := json.Marshal(history) path := filepath.Join(t.TempDir(), "history.json") diff --git a/backend/tools/collect_metrics.py b/backend/tools/collect_metrics.py index e721144..9f71bef 100644 --- a/backend/tools/collect_metrics.py +++ b/backend/tools/collect_metrics.py @@ -32,7 +32,7 @@ try: result = subprocess.run(['/usr/sbin/nft', '-j', 'list', 'counters', 'table', 'inet', 'mirrorouc_monitor'], capture_output=True, timeout=4, check=True) for item in json.loads(result.stdout).get('nftables', []): counter = item.get('counter', {}) - if counter.get('name') in ('https4', 'https6', 'rsync4', 'rsync6'): + if counter.get('name') in ('http4', 'http6', 'rsync4', 'rsync6'): counters[counter['name']] = counter['bytes'] except (OSError, ValueError, subprocess.SubprocessError): pass diff --git a/backend/tools/protocol_counters.py b/backend/tools/protocol_counters.py new file mode 100644 index 0000000..8b257e1 --- /dev/null +++ b/backend/tools/protocol_counters.py @@ -0,0 +1,197 @@ +#!/usr/bin/python3 +"""Maintain passive HTTP/S and rsync counters without resetting accumulated bytes. + +HTTP/S endpoints come from Nginx's HTTP configuration and tunasync HTTP(S) +upstreams. This is endpoint accounting, not packet inspection of arbitrary +encrypted protocols. Local proxy hops are excluded; both receive and transmit +traffic on the mirror host are counted, regardless of standard/custom ports. +""" +import argparse +import fnmatch +import hashlib +import json +import os +from pathlib import Path +import posixpath +import re +import subprocess +import tempfile +import tomllib +from urllib.parse import urlsplit + +TABLE = 'mirrorouc_monitor' +VERSION = 1 +COUNTERS = ('http4', 'http6', 'rsync4', 'rsync6') +TOKEN = re.compile(r'''"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\#[^\n]*|[{};]|[^\s{};\#]+''') + + +def directives(source, initial=()): + context, pending = list(initial), [] + for match in TOKEN.finditer(source): + token = match[0] + if token.startswith('#'): + continue + if token == '{': + context.append(pending[0] if pending else '') + pending = [] + elif token == '}': + if len(context) <= len(initial): + raise ValueError('Unbalanced Nginx configuration') + context.pop() + pending = [] + elif token == ';': + if pending: + yield tuple(context), pending + pending = [] + else: + pending.append(token.strip('\'"')) + + +def nginx_http_ports(dump): + parts = re.split(r'^# configuration file (.+):\s*$', dump, flags=re.M) + documents = dict(zip(parts[1::2], parts[2::2])) + if not documents: + documents = {'/etc/nginx/nginx.conf': dump} + entry = next(iter(documents)) + prefix = posixpath.dirname(entry) + todo, seen, ports = [(entry, ())], set(), set() + while todo: + file, initial = todo.pop() + if (file, initial) in seen: + continue + seen.add((file, initial)) + for context, args in directives(documents[file], initial): + if args[0] == 'include' and len(args) == 2: + pattern = posixpath.normpath(posixpath.join(prefix, args[1])) + todo.extend((name, context) for name in documents if fnmatch.fnmatchcase(name, pattern)) + elif args[0] == 'listen' and context == ('http', 'server') and len(args) > 1: + address = args[1] + if address.startswith('unix:') or 'quic' in args[2:]: + continue + value = address.rsplit(':', 1)[-1] + port = int(value) if value.isdecimal() else 80 + if not 1 <= port <= 65535: + raise ValueError('Invalid Nginx HTTP listener port') + ports.add(port) + return ports + + +def http_upstream_ports(worker_config): + # Include ordinary HTTP(S) clients as well as explicitly configured custom + # upstream ports. rsync/SSH/FTP URLs must not contribute ports to this set. + ports, seen, todo = {80, 443}, set(), [Path(worker_config)] + while todo: + path = todo.pop() + if path in seen or not path.exists(): + continue + seen.add(path) + doc = tomllib.loads(path.read_text()) + pattern = doc.get('include', {}).get('include_mirrors') + if pattern: + from glob import glob + todo.extend(Path(name) for name in glob(pattern)) + for mirror in doc.get('mirrors', []): + upstream = urlsplit(mirror.get('upstream', '')) + if upstream.scheme in ('http', 'https'): + ports.add(upstream.port or (443 if upstream.scheme == 'https' else 80)) + return ports + + +def shape(document): + def clean(value): + if isinstance(value, dict): + return {key: clean(item) for key, item in value.items() if key not in ('handle', 'packets', 'bytes')} + if isinstance(value, list): + return [clean(item) for item in value] + return value + objects = [clean(item) for item in document.get('nftables', []) if 'metainfo' not in item] + return hashlib.sha256(json.dumps(objects, sort_keys=True).encode()).hexdigest() + + +def transaction(current, listeners, upstreams, table=TABLE): + if not re.fullmatch(r'[a-zA-Z0-9_]+', table): + raise ValueError('Invalid counter table') + objects = current.get('nftables', []) + existing = lambda kind: {item[kind]['name'] for item in objects if kind in item} + lines = [] if existing('table') else [f'add table inet {table}'] + for name in COUNTERS: + if name not in existing('counter'): + lines.append(f'add counter inet {table} {name}') + for name, ports in (('http_listeners', listeners), ('http_upstreams', upstreams)): + if name not in existing('set'): + lines.append(f'add set inet {table} {name} {{ type inet_service; }}') + lines.append(f'flush set inet {table} {name}') + if ports: + if any(not isinstance(port, int) or not 1 <= port <= 65535 for port in ports): + raise ValueError('Invalid HTTP endpoint port') + lines.append(f'add element inet {table} {name} {{ ' + ', '.join(map(str, sorted(ports))) + ' }') + for direction, interface, local, remote in (('input', 'iifname', 'dport', 'sport'), ('output', 'oifname', 'sport', 'dport')): + if direction not in existing('chain'): + lines.append(f'add chain inet {table} {direction} {{ type filter hook {direction} priority 300; policy accept; }}') + lines.append(f'flush chain inet {table} {direction}') + lines.append(f'add rule inet {table} {direction} {interface} "lo" return') + for version in (4, 6): + for field, endpoint in ((local, 'http_listeners'), (remote, 'http_upstreams')): + # Return after accounting so a packet is counted exactly once. + lines.append(f'add rule inet {table} {direction} meta nfproto ipv{version} tcp {field} @{endpoint} counter name http{version} return') + for field in ('sport', 'dport'): + lines.append(f'add rule inet {table} {direction} meta nfproto ipv{version} tcp {field} 873 counter name rsync{version} return') + # Migrate the former HTTPS-only rules, preserving both rsync counters. + if 'account' in existing('chain'): + lines.extend((f'flush chain inet {table} account', f'delete chain inet {table} account')) + for name in ('https4', 'https6'): + if name in existing('counter'): + lines.append(f'delete counter inet {table} {name}') + return '\n'.join(lines) + '\n' + + +def read_table(nft): + result = subprocess.run([nft, '-j', '-a', 'list', 'table', 'inet', TABLE], capture_output=True, text=True, timeout=5) + if result.returncode: + if 'No such file or directory' in result.stderr or 'does not exist' in result.stderr: + return {} + raise RuntimeError('Unable to read protocol counters: ' + result.stderr.strip()) + return json.loads(result.stdout) + + +def ensure(args): + import fcntl + args.state.parent.mkdir(parents=True, exist_ok=True) + with (args.state.parent / '.protocol-counters.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + dump = subprocess.run([args.nginx, '-T'], capture_output=True, text=True, timeout=10, check=True).stdout + listeners = nginx_http_ports(dump) + if not listeners: + raise RuntimeError('No Nginx TCP HTTP listeners discovered; keeping existing accounting unchanged') + upstreams = http_upstream_ports(args.worker_config) + current = read_table(args.nft) + settings = {'version': VERSION, 'listeners': sorted(listeners), 'upstreams': sorted(upstreams)} + try: + previous = json.loads(args.state.read_text()) + except (OSError, ValueError): + previous = {} + if all(previous.get(key) == value for key, value in settings.items()) and previous.get('shape') == shape(current): + return + commands = transaction(current, listeners, upstreams) + for flags in (['-c'], []): + subprocess.run([args.nft, *flags, '-f', '-'], input=commands, capture_output=True, text=True, timeout=5, check=True) + settings['shape'] = shape(read_table(args.nft)) + fd, name = tempfile.mkstemp(dir=args.state.parent, prefix='.protocol-') + try: + with os.fdopen(fd, 'w') as stream: + json.dump(settings, stream, sort_keys=True) + os.chmod(name, 0o640) + os.replace(name, args.state) + finally: + if os.path.exists(name): + os.unlink(name) + print(json.dumps(settings), flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--nginx', default='/usr/sbin/nginx') + parser.add_argument('--nft', default='/usr/sbin/nft') + parser.add_argument('--worker-config', default='/etc/tunasync/worker.conf') + parser.add_argument('--state', type=Path, default=Path('/var/lib/mirrorouc/metrics/protocol-counters.json')) + ensure(parser.parse_args()) diff --git a/backend/tools/test_protocol_counters.py b/backend/tools/test_protocol_counters.py new file mode 100644 index 0000000..b93d582 --- /dev/null +++ b/backend/tools/test_protocol_counters.py @@ -0,0 +1,78 @@ +import copy +import importlib.util +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('protocol_counters', Path(__file__).with_name('protocol_counters.py')) +protocol = importlib.util.module_from_spec(spec) +spec.loader.exec_module(protocol) + + +class ProtocolTests(unittest.TestCase): + def test_http_contexts_and_included_custom_listeners(self): + dump = '''# configuration file /etc/nginx/nginx.conf: +events {} +http { + include /etc/nginx/sites-enabled/*; + server { listen 80; listen [::]:80; } +} +stream { include /etc/nginx/stream.d/*.conf; } +# configuration file /etc/nginx/sites-enabled/mirrors: +server { + listen "100.69.2.2:18080"; + listen [::]:443 ssl; + listen unix:/tmp/http.sock; + listen 9443 quic; + # listen 9999; + location ~ "^/test{2}#" { return 404; } + include ssl/cert.conf; +} +# configuration file /etc/nginx/ssl/cert.conf: +ssl_certificate /tmp/cert; +# configuration file /etc/nginx/stream.d/database.conf: +server { listen 5432; proxy_pass database; } +''' + self.assertEqual(protocol.nginx_http_ports(dump), {80, 443, 18080}) + + def test_upstream_schemes_and_custom_ports(self): + with tempfile.TemporaryDirectory() as name: + root = Path(name) + (root / 'worker.conf').write_text('[include]\ninclude_mirrors = "' + (root / '*.mirror').as_posix() + '"\n') + (root / 'repos.mirror').write_text('''[[mirrors]] +name = "http" +upstream = "http://example.invalid:19090/repo/" +[[mirrors]] +name = "https" +upstream = "https://example.invalid:8443/repo/" +[[mirrors]] +name = "rsync" +upstream = "rsync://example.invalid:1873/repo/" +''') + self.assertEqual(protocol.http_upstream_ports(root / 'worker.conf'), {80, 443, 19090, 8443}) + + def test_migration_keeps_existing_rsync_and_other_tables(self): + current = {'nftables': [{'table': {'name': protocol.TABLE}}, *({'counter': {'name': name, 'bytes': 12345}} for name in ('https4', 'https6', 'rsync4', 'rsync6')), *({'chain': {'name': name}} for name in ('input', 'output', 'account'))]} + commands = protocol.transaction(current, {80, 18080}, {80, 443, 19090}) + self.assertNotIn('flush ruleset', commands) + self.assertNotIn('flush table', commands) + self.assertNotIn('reset counter', commands) + for name in ('rsync4', 'rsync6'): + self.assertNotIn('add counter inet mirrorouc_monitor ' + name, commands) + self.assertNotIn('delete counter inet mirrorouc_monitor ' + name, commands) + self.assertIn('delete chain inet mirrorouc_monitor account', commands) + self.assertIn('meta nfproto ipv6 tcp sport @http_listeners counter name http6 return', commands) + self.assertIn('meta nfproto ipv4 tcp sport @http_upstreams counter name http4 return', commands) + self.assertIn('meta nfproto ipv4 tcp dport @http_upstreams counter name http4 return', commands) + + def test_counter_growth_does_not_reapply_rules(self): + before = {'nftables': [{'counter': {'name': 'http4', 'handle': 1, 'bytes': 100, 'packets': 1}}, {'rule': {'expr': [{'counter': 'http4'}]}}]} + after = copy.deepcopy(before) + after['nftables'][0]['counter'].update(bytes=200, packets=2, handle=9) + self.assertEqual(protocol.shape(before), protocol.shape(after)) + after['nftables'][1]['rule']['expr'][0]['counter'] = 'rsync4' + self.assertNotEqual(protocol.shape(before), protocol.shape(after)) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/check-monitoring.ts b/scripts/check-monitoring.ts index d49c64c..293aa1a 100644 --- a/scripts/check-monitoring.ts +++ b/scripts/check-monitoring.ts @@ -9,6 +9,9 @@ for (const hours of [1, 6, 24] as MonitoringRange[]) { assert.equal(snapshot.timestamps[0], end - hours * 3_600_000) assert.equal(snapshot.timestamps.at(-1), end) assert.equal(snapshot.panels.length, 6) + assert.equal(snapshot.panels[0]!.series[0]!.id, 'http4') + assert.equal(snapshot.panels[1]!.series[0]!.id, 'http6') + assert.equal(snapshot.panels[0]!.series[0]!.labelKey, 'monitor.series.http') for (const panel of snapshot.panels) { for (const series of panel.series) { assert.equal(series.values.length, snapshot.timestamps.length) @@ -34,4 +37,15 @@ assert.deepEqual(paths[1]!.plottedValues, [1, 3, null, 4, 5], 'Unknown stacked b assert.deepEqual(metricPaths([upper], 5, true, coordinate)[0]!.plottedValues, upper.values, 'Hiding a lower series must rebuild the stack') assert.deepEqual(metricStatistics([null, 0, 4, null]), { mean: 2, latest: null, max: 4 }, 'Missing samples must not become zero or stale latest values') assert.deepEqual(metricStatistics([null, null]), { mean: 0, latest: null, max: 0 }) +for (const family of [4, 6]) { + const panel = createMonitoringSnapshot(end, 24).panels[family === 4 ? 0 : 1]! + panel.series = [series(`http${family}`, [null, null, null, .1, .2]), series(`rsync${family}`, [.2, .3, .4, .5, .6])] + assert.equal(panel.stacked, false, 'Network protocols must use independent rates') + const networkPaths = metricPaths(panel.series, 5, panel.stacked, coordinate) + assert.deepEqual(networkPaths[0]!.plottedValues, [null, null, null, .1, .2], 'HTTP/S coordinates must match its rate, not the combined traffic') + assert.deepEqual(networkPaths[1]!.plottedValues, [.2, .3, .4, .5, .6], 'New HTTP/S collection must not hide existing rsync history') + assert.equal(networkPaths[1]!.line, 'M0,0.2 L1,0.3 L2,0.4 L3,0.5 L4,0.6') + const httpOnly = metricPaths([panel.series[0]!], 5, panel.stacked, coordinate) + assert.deepEqual(httpOnly[0]!.plottedValues, networkPaths[0]!.plottedValues, 'Hiding rsync must not move the HTTP/S curve') +} console.log('Monitoring check passed: deterministic windows, statistics, missing samples, line/fill gaps, stack limits and disk capacities.') diff --git a/src/locales/en-US/monitor.ts b/src/locales/en-US/monitor.ts index f0dc43d..af7bfe4 100644 --- a/src/locales/en-US/monitor.ts +++ b/src/locales/en-US/monitor.ts @@ -25,7 +25,7 @@ export default { cpu: 'CPU usage', memory: 'Memory usage', }, series: { - https: 'HTTPS', rsync: 'rsync', + http: 'HTTP/S', rsync: 'rsync', read: 'Read', write: 'Write', user: 'User', system: 'System', wait: 'I/O wait', idle: 'Idle', used: 'Used', cached: 'Cached', buffers: 'Buffers', free: 'Free', diff --git a/src/locales/zh-CN/monitor.ts b/src/locales/zh-CN/monitor.ts index e08c268..ff54cd7 100644 --- a/src/locales/zh-CN/monitor.ts +++ b/src/locales/zh-CN/monitor.ts @@ -25,7 +25,7 @@ export default { cpu: 'CPU 使用率', memory: '内存使用情况', }, series: { - https: 'HTTPS', rsync: 'rsync', + http: 'HTTP/S', rsync: 'rsync', read: '读取', write: '写入', user: '用户', system: '系统', wait: 'I/O 等待', idle: '空闲', used: '已使用', cached: '缓存', buffers: '缓冲区', free: '空闲', diff --git a/src/monitoring.ts b/src/monitoring.ts index a745562..807b06d 100644 --- a/src/monitoring.ts +++ b/src/monitoring.ts @@ -84,9 +84,9 @@ export function createMonitoringSnapshot(end: number, hours: MonitoringRange): M const cached = 22 + n(13, 600) * 3 const buffers = 2 + n(14, 600) return { - https4: 1.4 + activity * 4 + n(2) * 1.1, + http4: 1.4 + activity * 4 + n(2) * 1.1, rsync4: .2 + activity * .6 + Math.pow(n(3), 3) * .9, - https6: .4 + activity * 1.4 + n(5) * .4, + http6: .4 + activity * 1.4 + n(5) * .4, rsync6: .1 + activity * .3 + Math.pow(n(6), 3) * .4, readOps: 8 + activity * 34 + n(7) * 10, writeOps: .5 + n(8) * 1.5 + Math.pow(n(15, 45), 12) * 15, @@ -104,8 +104,8 @@ export function createMonitoringSnapshot(end: number, hours: MonitoringRange): M id, titleKey: `monitor.charts.${id}`, unit, maximum, stacked, series, }) return { timestamps, panels: [ - panel('ipv4', 'gigabits', 10, true, [series('https4', 'https', 'blue'), series('rsync4', 'rsync', 'violet')]), - panel('ipv6', 'gigabits', 4, true, [series('https6', 'https', 'blue'), series('rsync6', 'rsync', 'violet')]), + panel('ipv4', 'gigabits', 10, false, [series('http4', 'http', 'blue'), series('rsync4', 'rsync', 'violet')]), + panel('ipv6', 'gigabits', 4, false, [series('http6', 'http', 'blue'), series('rsync6', 'rsync', 'violet')]), panel('iops', 'operations', 60, false, [series('readOps', 'read', 'green'), series('writeOps', 'write', 'amber')]), panel('throughput', 'gigabytes', 3, false, [series('readBytes', 'read', 'blue'), series('writeBytes', 'write', 'violet')]), panel('cpu', 'percent', 100, true, [series('user', 'user', 'blue'), series('system', 'system', 'green'), series('wait', 'wait', 'amber'), series('idle', 'idle', 'chart-muted')]),