f8cef413d6
Integrate tunasync management, SQLite authentication and content settings, real monitoring and per-repository pull speeds. Add Chinese-only announcements and mirror help, editable related links, admin layout updates, deployment tools and tests.
125 lines
4.8 KiB
Python
125 lines
4.8 KiB
Python
#!/usr/bin/python3
|
|
"""Sample per-socket TCP receive counters and attribute worker processes to mirrors."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
PROC = Path('/proc')
|
|
MIRRORS = '/data/mirrors/'
|
|
OUTPUT = Path('/var/lib/mirrorouc/metrics/job-speeds.json')
|
|
NAME = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,127}$')
|
|
|
|
def mirror_name(value):
|
|
if not value.startswith(MIRRORS):
|
|
return None
|
|
name = value[len(MIRRORS):].split('/')[0]
|
|
return name if NAME.fullmatch(name) and name not in ('.', '..') else None
|
|
|
|
def process_jobs(proc=PROC):
|
|
jobs = {}
|
|
parents = {}
|
|
for entry in proc.iterdir():
|
|
if not entry.name.isdigit():
|
|
continue
|
|
try:
|
|
if '/tunasync-worker.service' not in (entry / 'cgroup').read_text():
|
|
continue
|
|
pid = int(entry.name)
|
|
status = (entry / 'stat').read_text().rsplit(') ', 1)[1].split()
|
|
parents[pid] = int(status[1])
|
|
arguments = (entry / 'cmdline').read_bytes().decode(errors='replace').split('\0')
|
|
candidates = {name for argument in arguments if (name := mirror_name(argument))}
|
|
if not candidates:
|
|
name = mirror_name(os.readlink(entry / 'cwd'))
|
|
if name:
|
|
candidates.add(name)
|
|
if len(candidates) == 1:
|
|
jobs[pid] = candidates.pop()
|
|
except (OSError, ValueError, IndexError):
|
|
continue
|
|
# git/ssh/curl helpers may inherit a job's parent without its destination argument.
|
|
for pid in parents:
|
|
ancestor = pid
|
|
for _ in range(16):
|
|
if ancestor in jobs:
|
|
jobs[pid] = jobs[ancestor]
|
|
break
|
|
ancestor = parents.get(ancestor, 0)
|
|
if not ancestor:
|
|
break
|
|
return jobs
|
|
|
|
def parse_sockets(output, jobs):
|
|
sockets = {}
|
|
header = None
|
|
for line in output.splitlines():
|
|
if line and not line[0].isspace():
|
|
header = line
|
|
continue
|
|
if header is None:
|
|
continue
|
|
identity = re.search(r'\bino:(\d+)\s+sk:([a-fA-F0-9]+)', header)
|
|
if not identity:
|
|
continue
|
|
pids = [int(value) for value in re.findall(r'\bpid=(\d+)', header)]
|
|
names = {jobs[pid] for pid in pids if pid in jobs}
|
|
received = re.search(r'\bbytes_received:(\d+)', line)
|
|
# The same inherited socket can list several PIDs: count its inode/cookie once.
|
|
sockets[':'.join(identity.groups())] = {'bytes': int(received.group(1)) if received else 0, 'job': names.pop() if len(names) == 1 else None}
|
|
header = None
|
|
return sockets
|
|
|
|
def rates(current, previous, active, elapsed):
|
|
result = {name: {'bytesPerSecond': 0.0, 'connections': 0} for name in active}
|
|
if previous is None or elapsed <= 0 or elapsed > 15:
|
|
return {}
|
|
for identity, socket in current.items():
|
|
name = socket['job']
|
|
if name is None:
|
|
continue
|
|
old = previous.get(identity)
|
|
delta = socket['bytes'] - old['bytes'] if old else socket['bytes']
|
|
if delta < 0: # A reset/reused socket must never produce a negative rate/spike.
|
|
continue
|
|
value = result.setdefault(name, {'bytesPerSecond': 0.0, 'connections': 0})
|
|
value['bytesPerSecond'] += delta / elapsed
|
|
value['connections'] += 1
|
|
return result
|
|
|
|
def write(value):
|
|
fd, temporary = tempfile.mkstemp(prefix='.speeds-', dir=OUTPUT.parent)
|
|
try:
|
|
with os.fdopen(fd, 'w') as stream:
|
|
json.dump(value, stream, allow_nan=False, separators=(',', ':'))
|
|
os.chmod(temporary, 0o640)
|
|
os.replace(temporary, OUTPUT)
|
|
finally:
|
|
if os.path.exists(temporary):
|
|
os.unlink(temporary)
|
|
|
|
def run():
|
|
previous = None
|
|
last = time.monotonic()
|
|
while True:
|
|
start = time.monotonic()
|
|
try:
|
|
jobs = process_jobs()
|
|
result = subprocess.run(['/usr/bin/ss', '-tinpeH', 'state', 'established'], capture_output=True, text=True, timeout=4, check=True, env={**os.environ, 'LC_ALL': 'C'})
|
|
current = parse_sockets(result.stdout, jobs)
|
|
sampled = time.monotonic()
|
|
elapsed = sampled - last
|
|
values = rates(current, previous, set(jobs.values()), elapsed)
|
|
write({'updatedAt': round(time.time() * 1000), 'interval': elapsed, 'available': previous is not None and elapsed <= 15, 'jobs': values})
|
|
previous, last = current, sampled
|
|
except (OSError, ValueError, subprocess.SubprocessError):
|
|
previous = None
|
|
write({'updatedAt': round(time.time() * 1000), 'interval': 2, 'available': False, 'jobs': {}})
|
|
time.sleep(max(.1, 2 - (time.monotonic() - start)))
|
|
|
|
if __name__ == '__main__':
|
|
run()
|