chore: simplify deployment and exclude local planning docs

This commit is contained in:
2026-11-01 21:15:40 +08:00
parent a5018faff7
commit 0cea76de81
23 changed files with 34 additions and 944 deletions
+124
View File
@@ -0,0 +1,124 @@
#!/usr/bin/python3
"""Sample per-socket TCP receive counters and attribute worker processes to mirrors."""
import json
import os
from pathlib import Path
import re
import subprocess
import tempfile
import time
PROC = Path('/proc')
MIRRORS = '/data/mirrors/'
OUTPUT = Path('/var/lib/mirrorouc/metrics/job-speeds.json')
NAME = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,127}$')
def mirror_name(value):
if not value.startswith(MIRRORS):
return None
name = value[len(MIRRORS):].split('/')[0]
return name if NAME.fullmatch(name) and name not in ('.', '..') else None
def process_jobs(proc=PROC):
jobs = {}
parents = {}
for entry in proc.iterdir():
if not entry.name.isdigit():
continue
try:
if '/tunasync-worker.service' not in (entry / 'cgroup').read_text():
continue
pid = int(entry.name)
status = (entry / 'stat').read_text().rsplit(') ', 1)[1].split()
parents[pid] = int(status[1])
arguments = (entry / 'cmdline').read_bytes().decode(errors='replace').split('\0')
candidates = {name for argument in arguments if (name := mirror_name(argument))}
if not candidates:
name = mirror_name(os.readlink(entry / 'cwd'))
if name:
candidates.add(name)
if len(candidates) == 1:
jobs[pid] = candidates.pop()
except (OSError, ValueError, IndexError):
continue
# git/ssh/curl helpers may inherit a job's parent without its destination argument.
for pid in parents:
ancestor = pid
for _ in range(16):
if ancestor in jobs:
jobs[pid] = jobs[ancestor]
break
ancestor = parents.get(ancestor, 0)
if not ancestor:
break
return jobs
def parse_sockets(output, jobs):
sockets = {}
header = None
for line in output.splitlines():
if line and not line[0].isspace():
header = line
continue
if header is None:
continue
identity = re.search(r'\bino:(\d+)\s+sk:([a-fA-F0-9]+)', header)
if not identity:
continue
pids = [int(value) for value in re.findall(r'\bpid=(\d+)', header)]
names = {jobs[pid] for pid in pids if pid in jobs}
received = re.search(r'\bbytes_received:(\d+)', line)
# The same inherited socket can list several PIDs: count its inode/cookie once.
sockets[':'.join(identity.groups())] = {'bytes': int(received.group(1)) if received else 0, 'job': names.pop() if len(names) == 1 else None}
header = None
return sockets
def rates(current, previous, active, elapsed):
result = {name: {'bytesPerSecond': 0.0, 'connections': 0} for name in active}
if previous is None or elapsed <= 0 or elapsed > 15:
return {}
for identity, socket in current.items():
name = socket['job']
if name is None:
continue
old = previous.get(identity)
delta = socket['bytes'] - old['bytes'] if old else socket['bytes']
if delta < 0: # A reset/reused socket must never produce a negative rate/spike.
continue
value = result.setdefault(name, {'bytesPerSecond': 0.0, 'connections': 0})
value['bytesPerSecond'] += delta / elapsed
value['connections'] += 1
return result
def write(value):
fd, temporary = tempfile.mkstemp(prefix='.speeds-', dir=OUTPUT.parent)
try:
with os.fdopen(fd, 'w') as stream:
json.dump(value, stream, allow_nan=False, separators=(',', ':'))
os.chmod(temporary, 0o640)
os.replace(temporary, OUTPUT)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def run():
previous = None
last = time.monotonic()
while True:
start = time.monotonic()
try:
jobs = process_jobs()
result = subprocess.run(['/usr/bin/ss', '-tinpeH', 'state', 'established'], capture_output=True, text=True, timeout=4, check=True, env={**os.environ, 'LC_ALL': 'C'})
current = parse_sockets(result.stdout, jobs)
sampled = time.monotonic()
elapsed = sampled - last
values = rates(current, previous, set(jobs.values()), elapsed)
write({'updatedAt': round(time.time() * 1000), 'interval': elapsed, 'available': previous is not None and elapsed <= 15, 'jobs': values})
previous, last = current, sampled
except (OSError, ValueError, subprocess.SubprocessError):
previous = None
write({'updatedAt': round(time.time() * 1000), 'interval': 2, 'available': False, 'jobs': {}})
time.sleep(max(.1, 2 - (time.monotonic() - start)))
if __name__ == '__main__':
run()
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/python3
"""Read Linux/NFS counters. A separate passive nftables table supplies protocol bytes."""
import json
import os
from pathlib import Path
import subprocess
import tempfile
import time
ROOT = Path(os.environ.get('MIRROROUC_METRICS_ROOT', '/var/lib/mirrorouc/metrics'))
ROOT.mkdir(parents=True, exist_ok=True)
path = ROOT / 'history.json'
try:
history = json.loads(path.read_text())
except (OSError, ValueError):
history = {'samples': []}
now = time.time()
cpu = list(map(int, Path('/proc/stat').read_text().splitlines()[0].split()[1:9]))
mem = {line.split(':')[0]: int(line.split()[1]) * 1024 for line in Path('/proc/meminfo').read_text().splitlines() if len(line.split()) > 1}
nfs = {}
for block in Path('/proc/self/mountstats').read_text().split('device '):
if ' mounted on /data ' not in block:
continue
for line in block.splitlines():
values = line.strip().split()
if values and values[0] in ('READ:', 'WRITE:'):
nfs[values[0][:-1]] = list(map(int, values[1:]))
counters = {}
try:
result = subprocess.run(['/usr/sbin/nft', '-j', 'list', 'counters', 'table', 'inet', 'mirrorouc_monitor'], capture_output=True, timeout=4, check=True)
for item in json.loads(result.stdout).get('nftables', []):
counter = item.get('counter', {})
if counter.get('name') in ('https4', 'https6', 'rsync4', 'rsync6'):
counters[counter['name']] = counter['bytes']
except (OSError, ValueError, subprocess.SubprocessError):
pass
previous = history.get('previous')
sample = {'time': round(now * 1000)}
if previous:
elapsed = max(.1, now - previous['time'])
delta = [max(0, a - b) for a, b in zip(cpu, previous['cpu'])]
total = sum(delta)
if total:
sample.update(user=(delta[0] + delta[1]) / total * 100, system=(delta[2] + delta[5] + delta[6] + delta[7]) / total * 100, wait=delta[4] / total * 100, idle=delta[3] / total * 100)
for key, value in counters.items():
old = previous.get('counters', {}).get(key)
if old is not None and value >= old:
sample[key] = (value - old) * 8 / elapsed / 1e9
for operation, prefix in [('READ', 'read'), ('WRITE', 'write')]:
a, b = nfs.get(operation), previous.get('nfs', {}).get(operation)
if a and b and a[0] >= b[0]:
sample[prefix + 'Ops'] = (a[0] - b[0]) / elapsed / 1000
byte_index = 4 if operation == 'READ' else 3
sample[prefix + 'Bytes'] = max(0, a[byte_index] - b[byte_index]) / elapsed / 1e9
cached = max(0, mem.get('Cached', 0) + mem.get('SReclaimable', 0) - mem.get('Shmem', 0))
free, buffers, total_mem = mem['MemFree'], mem.get('Buffers', 0), mem['MemTotal']
used = max(0, total_mem - free - cached - buffers)
sample.update(used=used / 2**30, cached=cached / 2**30, buffers=buffers / 2**30, free=free / 2**30)
if 'user' in sample:
history['samples'].append(sample)
disks = []
for mount in ['/data', '/']:
try:
disk = os.statvfs(mount)
disks.append({'name': mount, 'total': disk.f_blocks * disk.f_frsize / 1e12, 'used': (disk.f_blocks - disk.f_bfree) * disk.f_frsize / 1e12})
except OSError:
pass
history.update(previous={'time': now, 'cpu': cpu, 'nfs': nfs, 'counters': counters}, updatedAt=round(now * 1000), memoryTotal=mem['MemTotal'] / 2**30, disks=disks)
history['samples'] = [point for point in history['samples'] if point['time'] >= (now - 86400) * 1000][-8640:]
fd, temporary = tempfile.mkstemp(dir=ROOT, prefix='.metrics-')
try:
with os.fdopen(fd, 'w') as output:
json.dump(history, output, allow_nan=False, separators=(',', ':'))
output.flush()
os.fsync(output.fileno())
os.chmod(temporary, 0o640)
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
+178
View File
@@ -0,0 +1,178 @@
#!/usr/bin/python3
"""Root-only configuration writer, reachable only through a group-restricted Unix socket.
Independently verifies allowed TOML changes; never executes submitted commands.
"""
import copy
import glob
import grp
import hashlib
import json
import os
import re
import socketserver
import tempfile
import tomllib
import urllib.parse
import urllib.request
ROOT = '/etc/tunasync'
MAIN = ROOT + '/worker.conf'
INCLUDES = ROOT + '/mirrors.d'
SOCKET = '/run/mirrorouc-config/control.sock'
JOURNAL = '/var/lib/mirrorouc-config/pending.json'
IDENTIFIER = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,127}$')
def number(value, lower, upper):
return type(value) is int and lower <= value <= upper
def upstream(value):
if not isinstance(value, str) or len(value) >= 2048 or re.search(r'[\s`;$|<>\\\"\']', value):
return False
parsed = urllib.parse.urlsplit(value)
return parsed.scheme in ('rsync', 'https', 'http') and bool(parsed.hostname) and parsed.username is None
def sync_numbers(table):
return all(key not in table or number(table[key], low, high) for key, low, high in [('interval', 1, 525600), ('retry', 0, 20), ('timeout', 30, 1209600)])
def stripped_mirrors(doc):
result = copy.deepcopy(doc)
mirrors = result.get('mirrors', [])
assert isinstance(mirrors, list)
for mirror in mirrors:
for field in ('upstream', 'interval', 'retry', 'timeout'):
mirror.pop(field, None)
return result
def validate_new(doc, path):
assert set(doc) == {'mirrors'} and len(doc['mirrors']) == 1
mirror = doc['mirrors'][0]
assert set(mirror) <= {'name', 'provider', 'upstream', 'interval', 'retry', 'timeout', 'stage1_profile'}
assert IDENTIFIER.fullmatch(mirror['name']) and mirror['name'] not in ('.', '..')
assert path == INCLUDES + '/mirrorouc-' + mirror['name'] + '.conf'
assert mirror['provider'] in ('rsync', 'two-stage-rsync')
assert upstream(mirror['upstream']) and sync_numbers(mirror)
assert 'stage1_profile' not in mirror or mirror['stage1_profile'] == 'debian'
def validate(path, before, after):
assert path == MAIN or (os.path.dirname(path) == INCLUDES and os.path.basename(path).endswith('.conf'))
assert not os.path.islink(path)
old = tomllib.loads(before) if before is not None else None
new = tomllib.loads(after) if after is not None else None
assert old is not None or new is not None
if path == MAIN:
assert old is not None and new is not None
assert number(new['global'].get('concurrent'), 1, 64) and sync_numbers(new['global'])
for doc in (old, new):
for field in ('concurrent', 'interval', 'retry', 'timeout'):
doc['global'].pop(field, None)
assert old == new
elif old is None or new is None:
validate_new(new if new is not None else old, path)
else:
assert stripped_mirrors(old) == stripped_mirrors(new)
for previous, current in zip(old.get('mirrors', []), new.get('mirrors', [])):
if previous != current:
assert upstream(current['upstream']) and sync_numbers(current)
def read(path):
try:
with open(path, encoding='utf-8', newline='') as stream:
return stream.read()
except FileNotFoundError:
return None
def atomic(path, value):
if value is None:
os.unlink(path)
return
mode = os.stat(path).st_mode & 0o777 if os.path.exists(path) else 0o644
fd, temporary = tempfile.mkstemp(prefix='.mirrorouc-', dir=os.path.dirname(path))
try:
with os.fdopen(fd, 'w', encoding='utf-8', newline='') as stream:
stream.write(value)
stream.flush()
os.fsync(stream.fileno())
os.chmod(temporary, mode)
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def reload_worker():
worker = tomllib.loads(read(MAIN))['global']['name']
body = json.dumps({'worker_id': worker, 'mirror_id': '', 'cmd': 'reload', 'args': [], 'options': {}}).encode()
request = urllib.request.Request('http://127.0.0.1:12345/cmd', data=body, headers={'Content-Type': 'application/json'})
with urllib.request.urlopen(request, timeout=10) as response:
assert response.status == 200
def signature():
pattern = tomllib.loads(read(MAIN)).get('include', {}).get('include_mirrors', '')
assert pattern == INCLUDES + '/*.conf'
names = sorted(set([MAIN] + glob.glob(pattern)))
digest = hashlib.sha256()
for path in names:
assert not os.path.islink(path)
raw = read(path).encode()
digest.update(str(len(path.encode())).encode() + b':' + path.encode() + str(len(raw)).encode() + b':' + raw)
return digest.hexdigest()
def restore(changes):
for path, change in changes.items():
if change['before'] is None and not os.path.exists(path):
continue
atomic(path, change['before'])
reload_worker()
def apply(request):
changes = request['files']
assert isinstance(changes, dict) and 0 < len(changes) <= 20
for path, change in changes.items():
validate(path, change['before'], change['after'])
if read(path) != change['before']:
return 'CONFLICT'
if signature() != request['baseHash']:
return 'CONFLICT'
# Write-ahead journal survives a helper/process restart during a multi-file update.
atomic(JOURNAL, json.dumps(changes))
try:
for path, change in changes.items():
atomic(path, change['after'])
reload_worker()
except Exception:
try:
restore(changes)
except Exception:
return 'CONFIG_ROLLBACK_FAILED'
os.unlink(JOURNAL)
return 'CONFIG_APPLY_FAILED'
os.unlink(JOURNAL)
return ''
class Handler(socketserver.StreamRequestHandler):
def handle(self):
self.connection.settimeout(5)
try:
raw = self.rfile.readline(2 * 1024 * 1024 + 1)
assert len(raw) <= 2 * 1024 * 1024 and raw.endswith(b'\n')
if os.path.exists(JOURNAL):
code = 'CONFIG_ROLLBACK_FAILED'
else:
code = apply(json.loads(raw))
except (AssertionError, KeyError, TypeError, ValueError):
code = 'INVALID_INPUT'
except Exception:
code = 'CONFIG_APPLY_FAILED'
self.wfile.write(json.dumps({'code': code}).encode() + b'\n')
if __name__ == '__main__':
if os.path.exists(JOURNAL):
with open(JOURNAL) as stream:
restore(json.load(stream))
os.unlink(JOURNAL)
if os.path.exists(SOCKET):
os.unlink(SOCKET)
with socketserver.UnixStreamServer(SOCKET, Handler) as server:
os.chown(SOCKET, 0, grp.getgrnam('mirrorouc').gr_gid)
os.chmod(SOCKET, 0o660)
server.serve_forever()
+47
View File
@@ -0,0 +1,47 @@
import copy
import importlib.util
import json
import os
import tempfile
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('helper', os.path.join(os.path.dirname(__file__), 'config-helper.py'))
helper = importlib.util.module_from_spec(spec)
spec.loader.exec_module(helper)
class ValidationTests(unittest.TestCase):
def test_preserves_custom_commands(self):
before = '[[mirrors]]\nname="debian"\nprovider="rsync"\nupstream="rsync://old/debian"\ncommand="/usr/local/bin/parallel-rsync"\n'
after = before.replace('old/debian', 'new/debian') + 'interval=60\nretry=2\ntimeout=120\n'
helper.validate(helper.INCLUDES + '/debian.conf', before, after)
with self.assertRaises(AssertionError):
helper.validate(helper.INCLUDES + '/debian.conf', before, after.replace('parallel-rsync', 'other-command'))
def test_blocks_other_global_fields_and_paths(self):
before = '[global]\nname="worker"\nconcurrent=12\ninterval=60\nretry=2\ntimeout=120\n'
helper.validate(helper.MAIN, before, before.replace('concurrent=12', 'concurrent=8'))
for path, after in [('/etc/shadow', before), (helper.MAIN, before.replace('worker', 'other')), (helper.MAIN, before.replace('concurrent=12', 'concurrent=0'))]:
with self.assertRaises(AssertionError):
helper.validate(path, before, after)
def test_new_mirrors_and_injection(self):
value = '[[mirrors]]\nname="test"\nprovider="rsync"\nupstream="rsync://mirror/test"\ninterval=60\nretry=0\ntimeout=120\n'
helper.validate_new(helper.tomllib.loads(value), helper.INCLUDES + '/mirrorouc-test.conf')
for changed in [value + 'command="evil"\n', value.replace('rsync://mirror/test', 'rsync://mirror/`cmd`'), value.replace('retry=0', 'retry=-1')]:
with self.assertRaises(AssertionError):
helper.validate_new(helper.tomllib.loads(changed), helper.INCLUDES + '/mirrorouc-test.conf')
def test_reload_failure_restores_files(self):
with tempfile.TemporaryDirectory() as root:
path = root + '/worker.conf'
before = '[global]\nname="worker"\nconcurrent=12\ninterval=60\nretry=2\ntimeout=120\n'
helper.atomic(path, before)
changes = {path: {'before': before, 'after': before.replace('concurrent=12', 'concurrent=8')}}
with patch.object(helper, 'MAIN', path), patch.object(helper, 'JOURNAL', root + '/pending.json'), patch.object(helper, 'signature', return_value='base'), patch.object(helper, 'reload_worker', side_effect=[OSError('unavailable'), None]):
self.assertEqual(helper.apply({'files': changes, 'baseHash': 'base'}), 'CONFIG_APPLY_FAILED')
self.assertEqual(helper.read(path), before)
self.assertFalse(os.path.exists(root + '/pending.json'))
if __name__ == '__main__':
unittest.main()
+40
View File
@@ -0,0 +1,40 @@
import importlib.util
import os
from pathlib import Path
import tempfile
import unittest
spec = importlib.util.spec_from_file_location('speeds', os.path.join(os.path.dirname(__file__), 'collect_job_speeds.py'))
speeds = importlib.util.module_from_spec(spec)
spec.loader.exec_module(speeds)
class SpeedTests(unittest.TestCase):
def test_sockets_deduplicate_shared_pids_and_separate_repos(self):
data = '0 0 10.0.0.1:123 10.0.0.2:873 users:(("rsync",pid=1,fd=3),("rsync",pid=2,fd=3)) uid:9 ino:20 sk:abc <->\n\t cubic bytes_sent:4 bytes_received:1000\n0 0 10.0.0.1:124 10.0.0.2:873 users:(("rsync",pid=3,fd=3)) uid:9 ino:21 sk:abd <->\n\t cubic bytes_received:2000\n'
current = speeds.parse_sockets(data, {1: 'debian', 2: 'debian', 3: 'ubuntu'})
result = speeds.rates(current, {'20:abc': {'bytes': 400}, '21:abd': {'bytes': 1000}}, {'debian', 'ubuntu'}, 2)
self.assertEqual(result['debian'], {'bytesPerSecond': 300, 'connections': 1})
self.assertEqual(result['ubuntu']['bytesPerSecond'], 500)
def test_parallel_and_new_connections_add_without_startup_spike(self):
current = {'1:a': {'bytes': 1000, 'job': 'debian'}, '2:b': {'bytes': 500, 'job': 'debian'}, '3:c': {'bytes': 9000, 'job': None}}
self.assertEqual(speeds.rates(current, None, {'debian'}, 2), {})
result = speeds.rates(current, {'1:a': {'bytes': 800}}, {'debian', 'ubuntu'}, 2)
self.assertEqual(result['debian']['bytesPerSecond'], 350)
self.assertEqual(result['ubuntu']['bytesPerSecond'], 0)
self.assertEqual(speeds.rates(current, {}, {'debian'}, 20), {})
def test_process_ancestry_and_path_boundaries(self):
with tempfile.TemporaryDirectory() as root:
for pid, parent, command in [(1, 0, ['/usr/bin/python', '/worker']), (2, 1, ['rsync', '/data/mirrors/debian/']), (3, 2, ['ssh', 'host']), (4, 0, ['rsync', '/data/mirrors/ubuntu/'])]:
path = Path(root) / str(pid); path.mkdir()
(path/'stat').write_text(f'{pid} (process) S {parent} 0 0')
(path/'cgroup').write_text('0::/system.slice/tunasync-worker.service' if pid != 4 else '0::/system.slice/other.service')
(path/'cmdline').write_bytes('\0'.join(command).encode())
jobs = speeds.process_jobs(Path(root))
self.assertEqual(jobs, {2: 'debian', 3: 'debian'})
self.assertIsNone(speeds.mirror_name('/data/mirrors-other/secret'))
self.assertIsNone(speeds.mirror_name('/data/mirrors/../secret'))
if __name__ == '__main__':
unittest.main()