Files
OUCMirror-WebUI/backend/tools/collect_job_speeds.py
T

125 lines
4.8 KiB
Python

#!/usr/bin/python3
"""Sample per-socket TCP receive counters and attribute worker processes to mirrors."""
import json
import os
from pathlib import Path
import re
import subprocess
import tempfile
import time
PROC = Path('/proc')
MIRRORS = '/data/mirrors/'
OUTPUT = Path('/var/lib/mirrorouc/metrics/job-speeds.json')
NAME = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,127}$')
def mirror_name(value):
if not value.startswith(MIRRORS):
return None
name = value[len(MIRRORS):].split('/')[0]
return name if NAME.fullmatch(name) and name not in ('.', '..') else None
def process_jobs(proc=PROC):
jobs = {}
parents = {}
for entry in proc.iterdir():
if not entry.name.isdigit():
continue
try:
if '/tunasync-worker.service' not in (entry / 'cgroup').read_text():
continue
pid = int(entry.name)
status = (entry / 'stat').read_text().rsplit(') ', 1)[1].split()
parents[pid] = int(status[1])
arguments = (entry / 'cmdline').read_bytes().decode(errors='replace').split('\0')
candidates = {name for argument in arguments if (name := mirror_name(argument))}
if not candidates:
name = mirror_name(os.readlink(entry / 'cwd'))
if name:
candidates.add(name)
if len(candidates) == 1:
jobs[pid] = candidates.pop()
except (OSError, ValueError, IndexError):
continue
# git/ssh/curl helpers may inherit a job's parent without its destination argument.
for pid in parents:
ancestor = pid
for _ in range(16):
if ancestor in jobs:
jobs[pid] = jobs[ancestor]
break
ancestor = parents.get(ancestor, 0)
if not ancestor:
break
return jobs
def parse_sockets(output, jobs):
sockets = {}
header = None
for line in output.splitlines():
if line and not line[0].isspace():
header = line
continue
if header is None:
continue
identity = re.search(r'\bino:(\d+)\s+sk:([a-fA-F0-9]+)', header)
if not identity:
continue
pids = [int(value) for value in re.findall(r'\bpid=(\d+)', header)]
names = {jobs[pid] for pid in pids if pid in jobs}
received = re.search(r'\bbytes_received:(\d+)', line)
# The same inherited socket can list several PIDs: count its inode/cookie once.
sockets[':'.join(identity.groups())] = {'bytes': int(received.group(1)) if received else 0, 'job': names.pop() if len(names) == 1 else None}
header = None
return sockets
def rates(current, previous, active, elapsed):
result = {name: {'bytesPerSecond': 0.0, 'connections': 0} for name in active}
if previous is None or elapsed <= 0 or elapsed > 15:
return {}
for identity, socket in current.items():
name = socket['job']
if name is None:
continue
old = previous.get(identity)
delta = socket['bytes'] - old['bytes'] if old else socket['bytes']
if delta < 0: # A reset/reused socket must never produce a negative rate/spike.
continue
value = result.setdefault(name, {'bytesPerSecond': 0.0, 'connections': 0})
value['bytesPerSecond'] += delta / elapsed
value['connections'] += 1
return result
def write(value):
fd, temporary = tempfile.mkstemp(prefix='.speeds-', dir=OUTPUT.parent)
try:
with os.fdopen(fd, 'w') as stream:
json.dump(value, stream, allow_nan=False, separators=(',', ':'))
os.chmod(temporary, 0o640)
os.replace(temporary, OUTPUT)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def run():
previous = None
last = time.monotonic()
while True:
start = time.monotonic()
try:
jobs = process_jobs()
result = subprocess.run(['/usr/bin/ss', '-tinpeH', 'state', 'established'], capture_output=True, text=True, timeout=4, check=True, env={**os.environ, 'LC_ALL': 'C'})
current = parse_sockets(result.stdout, jobs)
sampled = time.monotonic()
elapsed = sampled - last
values = rates(current, previous, set(jobs.values()), elapsed)
write({'updatedAt': round(time.time() * 1000), 'interval': elapsed, 'available': previous is not None and elapsed <= 15, 'jobs': values})
previous, last = current, sampled
except (OSError, ValueError, subprocess.SubprocessError):
previous = None
write({'updatedAt': round(time.time() * 1000), 'interval': 2, 'available': False, 'jobs': {}})
time.sleep(max(.1, 2 - (time.monotonic() - start)))
if __name__ == '__main__':
run()