feat: add backend, admin console and live monitoring
Integrate tunasync management, SQLite authentication and content settings, real monitoring and per-repository pull speeds. Add Chinese-only announcements and mirror help, editable related links, admin layout updates, deployment tools and tests.
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/python3
|
||||
"""Sample per-socket TCP receive counters and attribute worker processes to mirrors."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
PROC = Path('/proc')
|
||||
MIRRORS = '/data/mirrors/'
|
||||
OUTPUT = Path('/var/lib/mirrorouc/metrics/job-speeds.json')
|
||||
NAME = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,127}$')
|
||||
|
||||
def mirror_name(value):
|
||||
if not value.startswith(MIRRORS):
|
||||
return None
|
||||
name = value[len(MIRRORS):].split('/')[0]
|
||||
return name if NAME.fullmatch(name) and name not in ('.', '..') else None
|
||||
|
||||
def process_jobs(proc=PROC):
|
||||
jobs = {}
|
||||
parents = {}
|
||||
for entry in proc.iterdir():
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
if '/tunasync-worker.service' not in (entry / 'cgroup').read_text():
|
||||
continue
|
||||
pid = int(entry.name)
|
||||
status = (entry / 'stat').read_text().rsplit(') ', 1)[1].split()
|
||||
parents[pid] = int(status[1])
|
||||
arguments = (entry / 'cmdline').read_bytes().decode(errors='replace').split('\0')
|
||||
candidates = {name for argument in arguments if (name := mirror_name(argument))}
|
||||
if not candidates:
|
||||
name = mirror_name(os.readlink(entry / 'cwd'))
|
||||
if name:
|
||||
candidates.add(name)
|
||||
if len(candidates) == 1:
|
||||
jobs[pid] = candidates.pop()
|
||||
except (OSError, ValueError, IndexError):
|
||||
continue
|
||||
# git/ssh/curl helpers may inherit a job's parent without its destination argument.
|
||||
for pid in parents:
|
||||
ancestor = pid
|
||||
for _ in range(16):
|
||||
if ancestor in jobs:
|
||||
jobs[pid] = jobs[ancestor]
|
||||
break
|
||||
ancestor = parents.get(ancestor, 0)
|
||||
if not ancestor:
|
||||
break
|
||||
return jobs
|
||||
|
||||
def parse_sockets(output, jobs):
|
||||
sockets = {}
|
||||
header = None
|
||||
for line in output.splitlines():
|
||||
if line and not line[0].isspace():
|
||||
header = line
|
||||
continue
|
||||
if header is None:
|
||||
continue
|
||||
identity = re.search(r'\bino:(\d+)\s+sk:([a-fA-F0-9]+)', header)
|
||||
if not identity:
|
||||
continue
|
||||
pids = [int(value) for value in re.findall(r'\bpid=(\d+)', header)]
|
||||
names = {jobs[pid] for pid in pids if pid in jobs}
|
||||
received = re.search(r'\bbytes_received:(\d+)', line)
|
||||
# The same inherited socket can list several PIDs: count its inode/cookie once.
|
||||
sockets[':'.join(identity.groups())] = {'bytes': int(received.group(1)) if received else 0, 'job': names.pop() if len(names) == 1 else None}
|
||||
header = None
|
||||
return sockets
|
||||
|
||||
def rates(current, previous, active, elapsed):
|
||||
result = {name: {'bytesPerSecond': 0.0, 'connections': 0} for name in active}
|
||||
if previous is None or elapsed <= 0 or elapsed > 15:
|
||||
return {}
|
||||
for identity, socket in current.items():
|
||||
name = socket['job']
|
||||
if name is None:
|
||||
continue
|
||||
old = previous.get(identity)
|
||||
delta = socket['bytes'] - old['bytes'] if old else socket['bytes']
|
||||
if delta < 0: # A reset/reused socket must never produce a negative rate/spike.
|
||||
continue
|
||||
value = result.setdefault(name, {'bytesPerSecond': 0.0, 'connections': 0})
|
||||
value['bytesPerSecond'] += delta / elapsed
|
||||
value['connections'] += 1
|
||||
return result
|
||||
|
||||
def write(value):
|
||||
fd, temporary = tempfile.mkstemp(prefix='.speeds-', dir=OUTPUT.parent)
|
||||
try:
|
||||
with os.fdopen(fd, 'w') as stream:
|
||||
json.dump(value, stream, allow_nan=False, separators=(',', ':'))
|
||||
os.chmod(temporary, 0o640)
|
||||
os.replace(temporary, OUTPUT)
|
||||
finally:
|
||||
if os.path.exists(temporary):
|
||||
os.unlink(temporary)
|
||||
|
||||
def run():
|
||||
previous = None
|
||||
last = time.monotonic()
|
||||
while True:
|
||||
start = time.monotonic()
|
||||
try:
|
||||
jobs = process_jobs()
|
||||
result = subprocess.run(['/usr/bin/ss', '-tinpeH', 'state', 'established'], capture_output=True, text=True, timeout=4, check=True, env={**os.environ, 'LC_ALL': 'C'})
|
||||
current = parse_sockets(result.stdout, jobs)
|
||||
sampled = time.monotonic()
|
||||
elapsed = sampled - last
|
||||
values = rates(current, previous, set(jobs.values()), elapsed)
|
||||
write({'updatedAt': round(time.time() * 1000), 'interval': elapsed, 'available': previous is not None and elapsed <= 15, 'jobs': values})
|
||||
previous, last = current, sampled
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
previous = None
|
||||
write({'updatedAt': round(time.time() * 1000), 'interval': 2, 'available': False, 'jobs': {}})
|
||||
time.sleep(max(.1, 2 - (time.monotonic() - start)))
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
Reference in New Issue
Block a user