Files

85 lines
3.8 KiB
Python

#!/usr/bin/python3
"""Read Linux/NFS counters. A separate passive nftables table supplies protocol bytes."""
import json
import os
from pathlib import Path
import subprocess
import tempfile
import time
ROOT = Path(os.environ.get('MIRROROUC_METRICS_ROOT', '/var/lib/mirrorouc/metrics'))
ROOT.mkdir(parents=True, exist_ok=True)
path = ROOT / 'history.json'
try:
history = json.loads(path.read_text())
except (OSError, ValueError):
history = {'samples': []}
now = time.time()
cpu = list(map(int, Path('/proc/stat').read_text().splitlines()[0].split()[1:9]))
mem = {line.split(':')[0]: int(line.split()[1]) * 1024 for line in Path('/proc/meminfo').read_text().splitlines() if len(line.split()) > 1}
nfs = {}
for block in Path('/proc/self/mountstats').read_text().split('device '):
if ' mounted on /data ' not in block:
continue
for line in block.splitlines():
values = line.strip().split()
if values and values[0] in ('READ:', 'WRITE:'):
nfs[values[0][:-1]] = list(map(int, values[1:]))
counters = {}
try:
result = subprocess.run(['/usr/sbin/nft', '-j', 'list', 'counters', 'table', 'inet', 'mirrorouc_monitor'], capture_output=True, timeout=4, check=True)
for item in json.loads(result.stdout).get('nftables', []):
counter = item.get('counter', {})
if counter.get('name') in ('http4', 'http6', 'rsync4', 'rsync6'):
counters[counter['name']] = counter['bytes']
except (OSError, ValueError, subprocess.SubprocessError):
pass
previous = history.get('previous')
sample = {'time': round(now * 1000)}
if previous:
elapsed = max(.1, now - previous['time'])
delta = [max(0, a - b) for a, b in zip(cpu, previous['cpu'])]
total = sum(delta)
if total:
sample.update(user=(delta[0] + delta[1]) / total * 100, system=(delta[2] + delta[5] + delta[6] + delta[7]) / total * 100, wait=delta[4] / total * 100, idle=delta[3] / total * 100)
for key, value in counters.items():
old = previous.get('counters', {}).get(key)
if old is not None and value >= old:
sample[key] = (value - old) * 8 / elapsed / 1e9
for operation, prefix in [('READ', 'read'), ('WRITE', 'write')]:
a, b = nfs.get(operation), previous.get('nfs', {}).get(operation)
if a and b and a[0] >= b[0]:
sample[prefix + 'Ops'] = (a[0] - b[0]) / elapsed / 1000
byte_index = 4 if operation == 'READ' else 3
sample[prefix + 'Bytes'] = max(0, a[byte_index] - b[byte_index]) / elapsed / 1e9
cached = max(0, mem.get('Cached', 0) + mem.get('SReclaimable', 0) - mem.get('Shmem', 0))
free, buffers, total_mem = mem['MemFree'], mem.get('Buffers', 0), mem['MemTotal']
used = max(0, total_mem - free - cached - buffers)
sample.update(used=used / 2**30, cached=cached / 2**30, buffers=buffers / 2**30, free=free / 2**30)
if 'user' in sample:
history['samples'].append(sample)
disks = []
for mount in ['/data', '/']:
try:
disk = os.statvfs(mount)
disks.append({'name': mount, 'total': disk.f_blocks * disk.f_frsize / 1e12, 'used': (disk.f_blocks - disk.f_bfree) * disk.f_frsize / 1e12})
except OSError:
pass
history.update(previous={'time': now, 'cpu': cpu, 'nfs': nfs, 'counters': counters}, updatedAt=round(now * 1000), memoryTotal=mem['MemTotal'] / 2**30, disks=disks)
history['samples'] = [point for point in history['samples'] if point['time'] >= (now - 86400) * 1000][-8640:]
fd, temporary = tempfile.mkstemp(dir=ROOT, prefix='.metrics-')
try:
with os.fdopen(fd, 'w') as output:
json.dump(history, output, allow_nan=False, separators=(',', ':'))
output.flush()
os.fsync(output.fileno())
os.chmod(temporary, 0o640)
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)