diff --git a/.gitignore b/.gitignore index 53bcb33..6345f7d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,9 @@ dist/ .env .env.* !.env.example + +backend/data/ +__pycache__/ +*.db +*.db-wal +*.db-shm diff --git a/README.md b/README.md index 4c6dc02..32187c6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ 中国海洋大学软件镜像源(MirrorOUC)网页界面 -本项目计划提供镜像检索、同步状态、使用帮助、软件下载及站点公告,并配套管理后台,用于维护站点内容、软件源配置和同步任务 +本项目提供镜像检索、同步状态、使用帮助、软件下载及站点公告,并配套管理后台,用于维护站点内容、软件源配置和同步任务 同步引擎采用 [TUNA tunasync](https://github.com/tuna/tunasync) @@ -19,7 +19,7 @@ npm run dev ```sh npm run check:i18n # 检查中英文语言键与插值参数 -npm run check:monitoring # 检查监控演示数据、时间窗口和资源总量 +npm run check:monitoring # 检查监控图表的数据夹具与资源总量约束 npm run build # TypeScript 检查与生产构建 npm run preview # 预览 dist 中的构建结果 ``` @@ -32,20 +32,48 @@ Vue 3 + TypeScript + Vite,使用 vue-i18n 管理界面文案。当前首页支 桌面端使用三列镜像列表与侧栏;手机端使用折叠导航。表格的后两列每 5 秒在「最近同步 + Size」与「下次同步 + Status」之间统一切换。表头每轮向上滚动一次,条目内容随机滚动 3~6 次后落定;使用系统的减少动态效果设置时直接显示最终值。页面隐藏时暂停轮换并取消未完成的动画。项目说明在悬浮提示及详情中查看。页脚使用宽幅分栏结构,在手机端纵向排列。新增页面也需兼顾手机端,并验证 320、390、768 和 1440 像素宽度下的中英文与明暗主题。 -当前为界面原型,15 个镜像、仓库容量及同步任务均为演示数据,尚未连接镜像服务器。示例配置使用 `example.invalid` 保留域名;下载入口前往项目官网。Markdown 公告发布、管理后台、真实目录与 tunasync 接入属于后续开发范围,详细方案见 [plan.md](plan.md)。 +前台已接入真实后端,不会在连接失败时切换为演示数据。首页每 15 秒获取镜像状态,连接中断时保留上一次数据并显示提示。镜像名称直接打开 `/mirrors//`,使用帮助和公告通过 Markdown 安全渲染。公告只展示已发布内容;公告标题/正文和镜像使用说明均维护一份中文内容,切换界面语言时显示同一版。界面文案、镜像介绍及站点设置继续支持多语言。 -`/status` 提供服务器监控与同步明细:IPv4 / IPv6 流量、两组磁盘容量、IOPS、吞吐量、CPU 和内存图表;可切换 1 / 6 / 24 小时范围,默认每 5 秒刷新图表,页面隐藏时暂停刷新。支持暂停图表刷新、悬停或方向键查看时间点、点击图例隐藏曲线,以及按名称或上游地址搜索同步任务。桌面端图表双列排列,手机端单列排列,同步表改为纵向条目。监控数据在 `src/monitoring.ts` 中生成,示例上游元数据在 `src/data.ts` 中,均未连接真实服务器。 +`/status` 展示服务器实测 IPv4 / IPv6 HTTPS、rsync 流量,NFS 磁盘容量、IOPS、吞吐量以及 CPU / 内存;支持 1 / 6 / 24 小时范围,图表每 10 秒刷新,页面隐藏时暂停。同步明细另提供按仓库汇总的上游 TCP 拉取速度,约每 2 秒采样与刷新,并适配手机端。统计值从完整采样计算,曲线最多取 241 个点。桌面端图表双列,手机端单列,同步明细在窄屏改为纵向布局。未采集到的指标不会冒充为零。 + +`/admin` 提供登录、Markdown 公告草稿/发布、镜像展示信息与使用帮助、任务命令和日志、全局/单个镜像调度配置、站点标识/头像/首页相关链接/页脚友链/下载项、管理员和公告编辑账户,以及操作记录。同步间隔遵循 tunasync 的原生语义:从上次同步结束后开始计算,单位为分钟。任务命令只表示已提交,最终状态以 Worker 为准。 + +## 后端与部署 + +后端使用 Go 标准库 HTTP、SQLite 和 TOML 解析器,不依赖数据库外部服务。详细部署、权限、备份和回退操作见 [deploy/README.md](deploy/README.md)。 + +```sh +# 需要 Go 1.27 或更高版本 +cd backend +go mod download +go test ./... +go vet ./... +go build -o mirrorouc . +# 首次运行生成初始管理员密码,妥善保存终端输出 +./mirrorouc -data ./data -init-admin admin +# 对接自己的 Manager 和 Worker;前端开发服务器已代理 /api 与 /uploads +./mirrorouc -data ./data -manager http://127.0.0.1:12345 -worker YOUR_WORKER_ID +``` + +生产构建通过 `-static ../dist` 或 `MIRROROUC_STATIC` 指定静态目录;`MIRROROUC_WORKER_CONFIG`、`MIRROROUC_LOGS`、`MIRROROUC_METRICS` 分别指定 Worker 配置、日志根目录和采样历史。缺少配置时,对应管理功能会显示不可用,不会启动假的同步引擎。管理页复用站点的明暗主题和语言选择。 + +当前服务器预览地址为 `http://100.69.2.2:8080`,管理入口 `/admin`,原有 80 端口下载服务保留。预览仅监听该 Tailnet 地址,正式域名与 HTTPS 尚未配置。 ## 源码约定 - `src/App.vue`:首页与弹窗交互。 - `src/style.css`:爱特官网同款明暗配色、布局、动画和移动端样式;色值来源与映射见 [DESIGN.md](DESIGN.md)。 -- `src/locales/zh-CN/`、`src/locales/en-US/`:界面文案及演示内容;新增文案通过语言键引用。 +- `src/locales/zh-CN/`、`src/locales/en-US/`:界面文案及数据夹具文案;新增文案通过语言键引用。 - 各语言的 `mirrors.ts` 定义表头、状态、相对时间、容量与公告日期模板,`formats.ts` 定义日期和数字格式;组件通过 `t`、`d`、`n` 使用,不自行生成界面文案。 -- `src/data.ts`:演示镜像与公告元数据。 +- `src/data.ts`:镜像类型与旧版界面数据夹具;生产页面通过 `src/usePublicData.ts` 和 `src/api.ts` 获取接口数据。 +- `src/components/AdminPage.vue`、`src/locales/admin-messages.ts`:管理表单与中英文文案。 +- `backend/`:鉴权、内容管理、tunasync 适配、配置预览与版本恢复、监控接口及集成测试。 +- `deploy/`:systemd、Nginx、实测采集器与受限配置写入工具。 - `src/components/StatusPage.vue`、`MonitoringChart.vue`:监控布局、图表与同步明细交互;`src/monitoring.ts` 定义监控数据结构和演示曲线,`src/useMirrorFormats.ts` 复用首页与状态页的时间、容量格式。 - `public/assets/`:页面实际使用的素材。 根目录 `素材/` 是本地素材目录,已加入 `.gitignore`,不得提交;`node_modules/`、`dist/`、`.local/` 和环境变量文件同样不纳入 Git。 中国海洋大学软件镜像源(MirrorOUC)由[中国海洋大学爱特工作室](https://www.itouc.cn/)承建。 + +首页侧栏“相关链接”在后台「站点设置 → 首页相关链接」管理,支持多语言名称、站内/站外网址、使用帮助入口、增删与排序。它使用独立的 `relatedLinks` 配置;页脚友情链接继续使用 `links`。保存站点设置后,首页下次刷新或轮询即可生效。 diff --git a/backend/app_test.go b/backend/app_test.go new file mode 100644 index 0000000..723f7ed --- /dev/null +++ b/backend/app_test.go @@ -0,0 +1,214 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type testClient struct { + t *testing.T + app *App + cookie *http.Cookie + csrf string +} + +func fixture(t *testing.T) *testClient { + t.Helper() + root := t.TempDir() + cfgDir := filepath.Join(root, "config") + os.MkdirAll(filepath.Join(cfgDir, "mirrors.d"), 0755) + worker := filepath.Join(cfgDir, "worker.conf") + inc := filepath.Join(cfgDir, "mirrors.d", "source.conf") + global := "[global]\nname='test'\nconcurrent=2\ninterval=60\nretry=2\ntimeout=600\n[include]\ninclude_mirrors=" + strconvQuote(filepath.ToSlash(filepath.Join(cfgDir, "mirrors.d", "*.conf"))) + "\n" + os.WriteFile(worker, []byte(global), 0644) + os.WriteFile(inc, []byte("[[mirrors]]\nname='alpine'\nprovider='rsync'\ncommand='/trusted/parallel-rsync'\nupstream='rsync://example.invalid/alpine/'\ninterval=60\nrsync_options=['--no-owner']\n"), 0644) + manager := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/workers/test/jobs": + respond(w, 200, []Job{{Name: "alpine", Worker: "test", Status: "failed", LastUpdate: time.Now().Add(-time.Hour), LastStarted: time.Now().Add(-time.Minute), Upstream: "https://user:secret@example.invalid/repo?token=secret", Size: "1.5G"}}) + case "/workers": + respond(w, 200, []Worker{{ID: "test"}}) + case "/cmd": + respond(w, 200, map[string]bool{"ok": true}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(manager.Close) + a, err := newApp(Config{Data: filepath.Join(root, "data"), Static: root, Manager: manager.URL, Worker: "test", WorkerConfig: worker, Logs: filepath.Join(root, "logs")}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { a.db.Close() }) + a.logger.SetOutput(io.Discard) + hash, err := hashPassword("test-only-password-123") + if err != nil { + t.Fatal(err) + } + _, err = a.db.Exec("INSERT INTO users(username,password,role,created) VALUES('admin',?,'admin',0),('editor',?,'editor',0)", hash, hash) + if err != nil { + t.Fatal(err) + } + a.refresh(context.Background()) + return &testClient{t: t, app: a} +} +func strconvQuote(s string) string { b, _ := json.Marshal(s); return string(b) } +func (c *testClient) call(method, path string, body any, csrf bool) *httptest.ResponseRecorder { + c.t.Helper() + var raw []byte + if body != nil { + raw, _ = json.Marshal(body) + } + r := httptest.NewRequest(method, "http://localhost"+path, bytes.NewReader(raw)) + r.Header.Set("Content-Type", "application/json") + if c.cookie != nil { + r.AddCookie(c.cookie) + } + if csrf { + r.Header.Set("X-CSRF-Token", c.csrf) + } + w := httptest.NewRecorder() + c.app.handler().ServeHTTP(w, r) + return w +} +func (c *testClient) login(user string) { + c.t.Helper() + w := c.call("POST", "/api/v1/auth/login", map[string]string{"username": user, "password": "test-only-password-123"}, false) + if w.Code != 200 { + c.t.Fatalf("login %d: %s", w.Code, w.Body.String()) + } + cookies := w.Result().Cookies() + if len(cookies) != 1 || !cookies[0].HttpOnly || cookies[0].SameSite != http.SameSiteStrictMode { + c.t.Fatal("unsafe session cookie") + } + c.cookie = cookies[0] + var result map[string]string + json.Unmarshal(w.Body.Bytes(), &result) + c.csrf = result["csrf"] +} +func status(t *testing.T, w *httptest.ResponseRecorder, want int) { + t.Helper() + if w.Code != want { + t.Fatalf("want %d got %d: %s", want, w.Code, w.Body.String()) + } +} +func TestAuthenticationAndRoles(t *testing.T) { + c := fixture(t) + status(t, c.call("GET", "/api/v1/admin/overview", nil, false), 401) + c.login("editor") + status(t, c.call("GET", "/api/v1/admin/config", nil, false), 403) + ann := Announcement{Title: "测试", Body: "草稿", Date: time.Now().UTC().Format(time.RFC3339)} + status(t, c.call("PUT", "/api/v1/admin/records/announcements/test", map[string]any{"data": ann, "revision": 0}, false), 403) + status(t, c.call("PUT", "/api/v1/admin/records/announcements/test", map[string]any{"data": ann, "revision": 0}, true), 200) + w := c.call("GET", "/api/v1/public", nil, false) + status(t, w, 200) + var public struct { + Announcements []Record `json:"announcements"` + Mirrors []PublicMirror `json:"mirrors"` + } + json.Unmarshal(w.Body.Bytes(), &public) + if len(public.Announcements) != 0 || len(public.Mirrors) != 1 || strings.Contains(public.Mirrors[0].Upstream, "secret") || strings.Contains(public.Mirrors[0].Upstream, "user") { + t.Fatal("draft or private upstream data exposed") + } + ann.Published = true + status(t, c.call("PUT", "/api/v1/admin/records/announcements/test", map[string]any{"data": ann, "revision": 1}, true), 200) + status(t, c.call("PUT", "/api/v1/admin/records/announcements/test", map[string]any{"data": ann, "revision": 1}, true), 409) + status(t, c.call("PUT", "/api/v1/auth/password", map[string]string{"current": "test-only-password-123", "password": "new-test-only-password-456"}, true), 200) + status(t, c.call("GET", "/api/v1/auth/session", nil, false), 401) +} +func TestCrossOriginAndManagerActions(t *testing.T) { + c := fixture(t) + c.login("admin") + r := httptest.NewRequest("POST", "http://localhost/api/v1/admin/jobs/alpine/start", nil) + r.AddCookie(c.cookie) + r.Header.Set("Origin", "https://attacker.invalid") + r.Header.Set("X-CSRF-Token", c.csrf) + w := httptest.NewRecorder() + c.app.handler().ServeHTTP(w, r) + status(t, w, 403) + status(t, c.call("POST", "/api/v1/admin/jobs/alpine/not-a-command", nil, true), 400) + status(t, c.call("POST", "/api/v1/admin/jobs/missing/start", nil, true), 404) + status(t, c.call("POST", "/api/v1/admin/jobs/alpine/start", nil, true), 202) +} +func TestConfigApplyRollbackAndConflict(t *testing.T) { + c := fixture(t) + c.login("admin") + initial, err := c.app.loadConfig() + if err != nil { + t.Fatal(err) + } + mirror := initial.mirrors[0] + mirror.Interval = 120 + w := c.call("POST", "/api/v1/admin/config/preview", map[string]any{"baseHash": initial.hash, "mirror": mirror}, true) + status(t, w, 201) + var draft configDraft + json.Unmarshal(w.Body.Bytes(), &draft) + status(t, c.call("POST", "/api/v1/admin/config/apply", map[string]string{"id": draft.ID}, true), 202) + updated, err := c.app.loadConfig() + if err != nil || updated.mirrors[0].Interval != 120 { + t.Fatal("config was not applied") + } + for _, f := range updated.files { + if strings.Contains(f.raw, "[[mirrors]]") && (!strings.Contains(f.raw, "/trusted/parallel-rsync") || !strings.Contains(f.raw, "--no-owner")) { + t.Fatal("custom configuration was lost") + } + } + status(t, c.call("POST", "/api/v1/admin/config/rollback", map[string]string{"id": draft.ID}, true), 202) + restored, err := c.app.loadConfig() + if err != nil || restored.hash != initial.hash { + t.Fatal("rollback did not restore exact bytes") + } + w = c.call("POST", "/api/v1/admin/config/preview", map[string]any{"baseHash": restored.hash, "mirror": mirror}, true) + status(t, w, 201) + json.Unmarshal(w.Body.Bytes(), &draft) + f, err := os.OpenFile(c.app.cfg.WorkerConfig, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + f.WriteString("\n# concurrent edit\n") + f.Close() + status(t, c.call("POST", "/api/v1/admin/config/apply", map[string]string{"id": draft.ID}, true), 409) +} +func TestConfigReloadFailureRestoresFiles(t *testing.T) { + c := fixture(t) + c.login("admin") + initial, _ := c.app.loadConfig() + mirror := initial.mirrors[0] + mirror.Interval = 90 + w := c.call("POST", "/api/v1/admin/config/preview", map[string]any{"baseHash": initial.hash, "mirror": mirror}, true) + status(t, w, 201) + var draft configDraft + json.Unmarshal(w.Body.Bytes(), &draft) + broken := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "unavailable", 503) })) + defer broken.Close() + c.app.cfg.Manager = broken.URL + status(t, c.call("POST", "/api/v1/admin/config/apply", map[string]string{"id": draft.ID}, true), 502) + restored, err := c.app.loadConfig() + if err != nil || restored.hash != initial.hash { + t.Fatal("failed reload did not restore configuration files") + } +} +func TestNullTimesSizesAndStaleCache(t *testing.T) { + if stamp(time.Time{}) != nil || parseSize("unknown") != nil || parseSize("NaN") != nil { + t.Fatal("missing values must remain missing") + } + if value := parseSize("1.5G"); value == nil || *value != 1.5*1024*1024*1024 { + t.Fatal("size conversion") + } + c := fixture(t) + c.app.cfg.Manager = "http://127.0.0.1:1" + c.app.refresh(context.Background()) + jobs, updated, code := c.app.cachedJobs() + if len(jobs) != 1 || updated == 0 || code == "" { + t.Fatal("last valid jobs were lost after upstream failure") + } +} diff --git a/backend/auth.go b/backend/auth.go new file mode 100644 index 0000000..ef67d72 --- /dev/null +++ b/backend/auth.go @@ -0,0 +1,217 @@ +package main + +import ( + "crypto/pbkdf2" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "errors" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +func randomToken() string { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + panic(err) + } + return base64.RawURLEncoding.EncodeToString(b) +} +func tokenHash(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:]) } +func hashPassword(password string) (string, error) { + if len(password) < 12 || len(password) > 256 { + return "", errors.New("INVALID_PASSWORD") + } + salt := make([]byte, 24) + if _, err := rand.Read(salt); err != nil { + return "", err + } + key, err := pbkdf2.Key(sha256.New, password, salt, 600000, 32) + if err != nil { + return "", err + } + return "pbkdf2$600000$" + base64.RawStdEncoding.EncodeToString(salt) + "$" + base64.RawStdEncoding.EncodeToString(key), nil +} +func verifyPassword(password, encoded string) bool { + p := strings.Split(encoded, "$") + if len(p) != 4 || p[0] != "pbkdf2" || len(password) > 256 { + return false + } + iterations, err := strconv.Atoi(p[1]) + if err != nil || iterations < 100000 || iterations > 1000000 { + return false + } + salt, err := base64.RawStdEncoding.DecodeString(p[2]) + if err != nil { + return false + } + want, err := base64.RawStdEncoding.DecodeString(p[3]) + if err != nil || len(want) != 32 { + return false + } + got, err := pbkdf2.Key(sha256.New, password, salt, iterations, 32) + return err == nil && subtle.ConstantTimeCompare(got, want) == 1 +} +func (a *App) sameOrigin(r *http.Request) bool { + if r.Header.Get("Sec-Fetch-Site") == "cross-site" { + return false + } + if origin := r.Header.Get("Origin"); origin != "" { + u, err := url.Parse(origin) + return err == nil && (u.Scheme == "http" || u.Scheme == "https") && strings.EqualFold(u.Host, r.Host) + } + return true +} +func (a *App) currentSession(r *http.Request) (Session, error) { + var s Session + c, err := r.Cookie("mirrorouc_session") + if err != nil { + return s, err + } + err = a.db.QueryRow("SELECT users.id,username,role,csrf FROM sessions JOIN users ON users.id=sessions.user_id WHERE token=? AND expires>?", tokenHash(c.Value), time.Now().Unix()).Scan(&s.UserID, &s.Username, &s.Role, &s.CSRF) + return s, err +} +func (a *App) require(w http.ResponseWriter, r *http.Request, admin bool) (Session, bool) { + s, err := a.currentSession(r) + if err != nil { + fail(w, 401, "UNAUTHORIZED") + return s, false + } + if admin && s.Role != "admin" { + fail(w, 403, "FORBIDDEN") + return s, false + } + if r.Method != "GET" && r.Method != "HEAD" { + if !a.sameOrigin(r) || subtle.ConstantTimeCompare([]byte(r.Header.Get("X-CSRF-Token")), []byte(s.CSRF)) != 1 { + fail(w, 403, "CSRF_INVALID") + return s, false + } + } + return s, true +} +func (a *App) session(w http.ResponseWriter, r *http.Request) { + s, err := a.currentSession(r) + if err != nil { + fail(w, 401, "UNAUTHORIZED") + return + } + respond(w, 200, map[string]any{"username": s.Username, "role": s.Role, "csrf": s.CSRF}) +} +func (a *App) login(w http.ResponseWriter, r *http.Request) { + if !a.sameOrigin(r) { + fail(w, 403, "CSRF_INVALID") + return + } + address, _, _ := net.SplitHostPort(r.RemoteAddr) + a.loginMu.Lock() + attempt := a.attempts[address] + if time.Since(attempt.at) > time.Minute { + attempt = loginAttempt{at: time.Now()} + } + attempt.count++ + a.attempts[address] = attempt + for key, value := range a.attempts { + if time.Since(value.at) > 10*time.Minute { + delete(a.attempts, key) + } + } + a.loginMu.Unlock() + if attempt.count > 10 { + w.Header().Set("Retry-After", "60") + fail(w, 429, "RATE_LIMITED") + return + } + var body struct { + Username string `json:"username"` + Password string `json:"password"` + } + if !readJSON(w, r, &body) { + return + } + var s Session + var encoded string + err := a.db.QueryRow("SELECT id,username,role,password FROM users WHERE username=?", body.Username).Scan(&s.UserID, &s.Username, &s.Role, &encoded) + if err != nil { + encoded = a.dummyPassword + } + valid := verifyPassword(body.Password, encoded) + if err != nil || !valid { + a.audit(Session{Username: body.Username}, "login", "", "denied") + fail(w, 401, "INVALID_CREDENTIALS") + return + } + token, csrf := randomToken(), randomToken() + expires := time.Now().Add(12 * time.Hour) + if _, err = a.db.Exec("INSERT INTO sessions(token,user_id,csrf,expires) VALUES(?,?,?,?)", tokenHash(token), s.UserID, csrf, expires.Unix()); err != nil { + a.internal(w, err) + return + } + a.db.Exec("DELETE FROM sessions WHERE expires= 1 && interval <= 525600 && retry >= 0 && retry <= 20 && timeout >= 30 && timeout <= 1209600 +} +func validUpstream(value string) bool { + u, err := url.Parse(value) + return err == nil && (u.Scheme == "rsync" || u.Scheme == "https" || u.Scheme == "http") && u.Host != "" && u.User == nil && !strings.ContainsAny(value, "\r\n\t `;$|<>\\\"'") && len(value) < 2048 +} +func (a *App) configPreview(w http.ResponseWriter, r *http.Request) { + s, ok := a.require(w, r, true) + if !ok { + return + } + var body struct { + BaseHash string `json:"baseHash"` + Global *ConfigNumbers `json:"global"` + Mirror *ConfigMirror `json:"mirror"` + } + if !readJSON(w, r, &body) { + return + } + a.configMu.Lock() + defer a.configMu.Unlock() + c, err := a.loadConfig() + if err != nil { + fail(w, 503, "CONFIG_UNAVAILABLE") + return + } + if body.BaseHash != c.hash { + fail(w, 409, "CONFLICT") + return + } + changed := map[string]bool{} + if body.Global != nil { + g := body.Global + if g.Concurrent < 1 || g.Concurrent > 64 || !validNumbers(g.Interval, g.Retry, g.Timeout) { + fail(w, 400, "INVALID_INPUT") + return + } + if *g != c.global { + f := c.files[a.cfg.WorkerConfig] + m := asMap(f.doc["global"]) + m["concurrent"], m["interval"], m["retry"], m["timeout"] = g.Concurrent, g.Interval, g.Retry, g.Timeout + f.doc["global"] = m + changed[a.cfg.WorkerConfig] = true + } + } + if body.Mirror != nil { + m := body.Mirror + if !validName(m.Name) || !validUpstream(m.Upstream) || !validNumbers(m.Interval, m.Retry, m.Timeout) { + fail(w, 400, "INVALID_INPUT") + return + } + var current *ConfigMirror + for _, entry := range c.mirrors { + if entry.Name == m.Name { + copy := entry + current = © + break + } + } + if current != nil { + if current.Provider != m.Provider { + fail(w, 400, "INVALID_INPUT") + return + } + if current.Upstream != m.Upstream || current.Interval != m.Interval || current.Retry != m.Retry || current.Timeout != m.Timeout { + f := c.files[current.Source] + for _, table := range asTables(f.doc["mirrors"]) { + if text(table["name"]) == m.Name { + table["upstream"], table["interval"], table["retry"], table["timeout"] = m.Upstream, m.Interval, m.Retry, m.Timeout + } + } + changed[current.Source] = true + } + } else { + if c.includeDir == "" || (m.Provider != "rsync" && m.Provider != "two-stage-rsync") { + fail(w, 400, "INVALID_INPUT") + return + } + path := filepath.Join(c.includeDir, "mirrorouc-"+m.Name+".conf") + if _, err = os.Lstat(path); !os.IsNotExist(err) { + fail(w, 409, "CONFLICT") + return + } + entry := map[string]any{"name": m.Name, "provider": m.Provider, "upstream": m.Upstream, "interval": m.Interval, "retry": m.Retry, "timeout": m.Timeout} + if m.Provider == "two-stage-rsync" { + entry["stage1_profile"] = "debian" + } + c.files[path] = configFile{"", map[string]any{"mirrors": []map[string]any{entry}}} + changed[path] = true + } + } + if len(changed) == 0 { + fail(w, 400, "NO_CHANGES") + return + } + files := map[string]fileChange{} + for path := range changed { + f := c.files[path] + var b bytes.Buffer + if err = toml.NewEncoder(&b).Encode(f.doc); err != nil { + a.internal(w, err) + return + } + after := b.String() + var before *string + if _, err = os.Stat(path); err == nil { + old := f.raw + before = &old + } + files[path] = fileChange{before, &after} + } + id := tokenHash(randomToken())[:24] + raw, _ := json.Marshal(files) + now := time.Now().UnixMilli() + _, err = a.db.Exec("INSERT INTO config_versions(id,files,base_hash,status,created,username) VALUES(?,?,?,'draft',?,?)", id, string(raw), c.hash, now, s.Username) + if err != nil { + a.internal(w, err) + return + } + a.audit(s, "config_preview", id, "draft") + respond(w, 201, configDraft{id, files, c.hash, "draft", now, s.Username}) +} +func atomicFile(path string, content *string) error { + if content == nil { + return os.Remove(path) + } + mode := os.FileMode(0660) + if info, err := os.Stat(path); err == nil { + mode = info.Mode().Perm() + } + f, err := os.CreateTemp(filepath.Dir(path), ".mirrorouc-*.tmp") + if err != nil { + return err + } + tmp := f.Name() + defer os.Remove(tmp) + if err = f.Chmod(mode); err == nil { + _, err = f.WriteString(*content) + } + if err == nil { + err = f.Sync() + } + if e := f.Close(); err == nil { + err = e + } + if err != nil { + return err + } + return os.Rename(tmp, path) +} +func (a *App) readVersion(id string) (configDraft, error) { + var v configDraft + var raw string + v.ID = id + err := a.db.QueryRow("SELECT files,base_hash,status,created,username FROM config_versions WHERE id=?", id).Scan(&raw, &v.BaseHash, &v.Status, &v.Created, &v.Username) + if err != nil { + return v, err + } + err = json.Unmarshal([]byte(raw), &v.Files) + return v, err +} +func (a *App) applyVersion(r *http.Request, v configDraft, s Session) (string, error) { + c, err := a.loadConfig() + if err != nil { + return "CONFIG_UNAVAILABLE", err + } + if c.hash != v.BaseHash { + return "CONFLICT", errConflict + } + for path, change := range v.Files { + raw, e := os.ReadFile(path) + if change.Before == nil { + if !os.IsNotExist(e) { + return "CONFLICT", errConflict + } + } else if e != nil || string(raw) != *change.Before { + return "CONFLICT", errConflict + } + } + if a.cfg.ConfigSocket != "" { + conn, err := net.DialTimeout("unix", a.cfg.ConfigSocket, 5*time.Second) + if err != nil { + return "CONFIG_UNAVAILABLE", err + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(28 * time.Second)) + if err = json.NewEncoder(conn).Encode(v); err != nil { + return "CONFIG_STATE_UNCERTAIN", err + } + var result struct { + Code string `json:"code"` + } + if err = json.NewDecoder(bufio.NewReader(conn)).Decode(&result); err != nil { + return "CONFIG_STATE_UNCERTAIN", err + } + if result.Code != "" { + a.db.Exec("UPDATE config_versions SET status='failed' WHERE id=?", v.ID) + return result.Code, errors.New(result.Code) + } + if _, err = a.db.Exec("UPDATE config_versions SET status='applied' WHERE id=?", v.ID); err != nil { + return "CONFIG_STATE_UNCERTAIN", err + } + a.audit(s, "config_apply", v.ID, "submitted") + return "", nil + } + written := []string{} + restore := func() error { + var result error + for i := len(written) - 1; i >= 0; i-- { + path := written[i] + if e := atomicFile(path, v.Files[path].Before); e != nil { + result = e + } + } + return result + } + for _, path := range sortedKeys(v.Files) { + if err = atomicFile(path, v.Files[path].After); err != nil { + if e := restore(); e != nil { + return "CONFIG_ROLLBACK_FAILED", e + } + return "CONFIG_APPLY_FAILED", err + } + written = append(written, path) + } + if _, err = a.loadConfig(); err == nil { + err = a.sendCommand(r.Context(), "", "reload") + } + if err != nil { + restoreErr := restore() + if restoreErr == nil { + restoreErr = a.sendCommand(r.Context(), "", "reload") + } + a.db.Exec("UPDATE config_versions SET status='failed' WHERE id=?", v.ID) + if restoreErr != nil { + return "CONFIG_ROLLBACK_FAILED", restoreErr + } + return "CONFIG_APPLY_FAILED", err + } + if _, err = a.db.Exec("UPDATE config_versions SET status='applied' WHERE id=?", v.ID); err != nil { + return "CONFIG_STATE_UNCERTAIN", err + } + a.audit(s, "config_apply", v.ID, "submitted") + return "", nil +} +func (a *App) configApply(w http.ResponseWriter, r *http.Request) { + s, ok := a.require(w, r, true) + if !ok { + return + } + var body struct { + ID string `json:"id"` + } + if !readJSON(w, r, &body) { + return + } + a.configMu.Lock() + defer a.configMu.Unlock() + v, err := a.readVersion(body.ID) + if err != nil { + fail(w, 404, "NOT_FOUND") + return + } + if v.Status != "draft" { + fail(w, 409, "CONFLICT") + return + } + code, err := a.applyVersion(r, v, s) + if err != nil { + a.logger.Printf("config apply: %v", err) + a.audit(s, "config_apply", v.ID, code) + status := 502 + if code == "CONFLICT" { + status = 409 + } + fail(w, status, code) + return + } + respond(w, 202, map[string]string{"status": "submitted"}) +} +func (a *App) configVersions(w http.ResponseWriter, r *http.Request) { + if _, ok := a.require(w, r, true); !ok { + return + } + rows, err := a.db.Query("SELECT id,base_hash,status,created,username FROM config_versions ORDER BY created DESC LIMIT 50") + if err != nil { + a.internal(w, err) + return + } + defer rows.Close() + out := []configDraft{} + for rows.Next() { + var v configDraft + if err = rows.Scan(&v.ID, &v.BaseHash, &v.Status, &v.Created, &v.Username); err != nil { + a.internal(w, err) + return + } + out = append(out, v) + } + respond(w, 200, out) +} +func (a *App) configRollback(w http.ResponseWriter, r *http.Request) { + s, ok := a.require(w, r, true) + if !ok { + return + } + var body struct { + ID string `json:"id"` + } + if !readJSON(w, r, &body) { + return + } + a.configMu.Lock() + defer a.configMu.Unlock() + old, err := a.readVersion(body.ID) + if err != nil { + fail(w, 404, "NOT_FOUND") + return + } + if old.Status != "applied" { + fail(w, 409, "CONFLICT") + return + } + c, err := a.loadConfig() + if err != nil { + fail(w, 503, "CONFIG_UNAVAILABLE") + return + } + reverse := map[string]fileChange{} + for path, change := range old.Files { + raw, e := os.ReadFile(path) + if change.After == nil { + if !os.IsNotExist(e) { + fail(w, 409, "CONFLICT") + return + } + } else if e != nil || string(raw) != *change.After { + fail(w, 409, "CONFLICT") + return + } + reverse[path] = fileChange{change.After, change.Before} + } + id := tokenHash(randomToken())[:24] + now := time.Now().UnixMilli() + raw, _ := json.Marshal(reverse) + v := configDraft{id, reverse, c.hash, "draft", now, s.Username} + if _, err = a.db.Exec("INSERT INTO config_versions(id,files,base_hash,status,created,username) VALUES(?,?,?,'draft',?,?)", id, string(raw), c.hash, now, s.Username); err != nil { + a.internal(w, err) + return + } + code, err := a.applyVersion(r, v, s) + if err != nil { + a.audit(s, "config_rollback", old.ID, code) + fail(w, 502, code) + return + } + a.audit(s, "config_rollback", old.ID, "submitted") + respond(w, 202, map[string]string{"status": "submitted"}) +} diff --git a/backend/content.go b/backend/content.go new file mode 100644 index 0000000..ae00489 --- /dev/null +++ b/backend/content.go @@ -0,0 +1,402 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "net/http" + "net/mail" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" +) + +var identifier = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,127}$`) + +func validName(value string) bool { + return identifier.MatchString(value) && value != "." && value != ".." +} +func validText(value Localized, required bool, limit int) bool { + has := false + if len(value) > 20 { + return false + } + for locale, text := range value { + if len(locale) > 48 || !localeIdentifier.MatchString(locale) { + return false + } + if len(text) > limit { + return false + } + if strings.TrimSpace(text) != "" { + has = true + } + } + return !required || has +} + +var localeIdentifier = regexp.MustCompile(`^[a-zA-Z]{2,3}(?:-[a-zA-Z0-9]{2,8})*$`) + +func webURL(value string, empty bool) bool { + if empty && value == "" { + return true + } + u, e := url.Parse(value) + return e == nil && (u.Scheme == "http" || u.Scheme == "https") && u.Host != "" && u.User == nil && len(value) < 2048 +} +func navigationURL(value string) bool { + if webURL(value, false) { + return true + } + if !strings.HasPrefix(value, "/") || strings.HasPrefix(value, "//") || len(value) >= 2048 || strings.ContainsAny(value, "\\\r\n\t") { + return false + } + u, err := url.Parse(value) + return err == nil && u.Scheme == "" && u.Host == "" && u.User == nil +} +func imageURL(value string) bool { + if strings.HasPrefix(value, "/assets/") || strings.HasPrefix(value, "/uploads/") { + return !strings.Contains(value, "..") && !strings.ContainsAny(value, "\\?#") + } + u, e := url.Parse(value) + return e == nil && u.Scheme == "https" && u.Host != "" && u.User == nil +} +func strict(raw []byte, value any) bool { + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + return d.Decode(value) == nil +} +func (a *App) listRecords(w http.ResponseWriter, r *http.Request) { + kind := r.PathValue("kind") + if kind != "announcements" && kind != "mirrors" && kind != "settings" { + fail(w, 404, "NOT_FOUND") + return + } + if _, ok := a.require(w, r, kind != "announcements"); !ok { + return + } + records, err := a.records(kind) + if err != nil { + a.internal(w, err) + return + } + respond(w, 200, records) +} +func (a *App) putRecord(w http.ResponseWriter, r *http.Request) { + kind, id := r.PathValue("kind"), r.PathValue("id") + if !validName(id) || (kind != "announcements" && kind != "mirrors" && kind != "settings") { + fail(w, 400, "INVALID_INPUT") + return + } + s, ok := a.require(w, r, kind != "announcements") + if !ok { + return + } + var body struct { + Data json.RawMessage `json:"data"` + Revision int `json:"revision"` + } + if !readJSON(w, r, &body) { + return + } + var value any + switch kind { + case "announcements": + var d Announcement + if !strict(body.Data, &d) || strings.TrimSpace(d.Title) == "" || len(d.Title) > 300 || len(d.Body) > 250000 || (d.Published && strings.TrimSpace(d.Body) == "") { + fail(w, 400, "INVALID_INPUT") + return + } + if d.Date == "" { + d.Date = time.Now().UTC().Format(time.RFC3339) + } + if _, err := time.Parse(time.RFC3339, d.Date); err != nil { + fail(w, 400, "INVALID_INPUT") + return + } + value = d + case "mirrors": + var d MirrorMetadata + if !strict(body.Data, &d) || d.Name != id || !validText(d.Description, false, 1000) || len(d.Help) > 250000 || !webURL(d.Website, true) || (d.Category != "os" && d.Category != "language" && d.Category != "other") { + fail(w, 400, "INVALID_INPUT") + return + } + value = d + case "settings": + var d Settings + if id != "site" || !strict(body.Data, &d) || !validText(d.Name, true, 200) || !validText(d.Description, false, 2000) || !validText(d.Address, false, 2000) || !imageURL(d.LogoLight) || !imageURL(d.LogoDark) || !imageURL(d.Favicon) || !imageURL(d.Avatar) || len(d.Links) > 50 || len(d.RelatedLinks) > 50 || len(d.Downloads) > 100 { + fail(w, 400, "INVALID_INPUT") + return + } + var fields map[string]json.RawMessage + json.Unmarshal(body.Data, &fields) + if _, provided := fields["relatedLinks"]; !provided { + // Preserve saved sidebar changes when an older open form submits without this field. + current, err := a.record("settings", "site") + var existing Settings + if err != nil || decodeData(current.Data, &existing) != nil { + fail(w, 409, "CONFLICT") + return + } + d.RelatedLinks = existing.RelatedLinks + } + if d.RelatedLinks == nil { + d.RelatedLinks = []RelatedLink{} + } + for _, link := range d.RelatedLinks { + if !validText(link.Title, true, 200) || (link.Kind != "help" && link.Kind != "url") || (link.Kind == "help" && link.URL != "") || (link.Kind == "url" && !navigationURL(link.URL)) { + fail(w, 400, "INVALID_INPUT") + return + } + } + if d.Email != "" { + if parsed, err := mail.ParseAddress(d.Email); err != nil || parsed.Address != d.Email { + fail(w, 400, "INVALID_INPUT") + return + } + } + for _, link := range d.Links { + if !validText(link.Title, true, 200) || !webURL(link.URL, false) { + fail(w, 400, "INVALID_INPUT") + return + } + } + for _, download := range d.Downloads { + if strings.TrimSpace(download.Name) == "" || len(download.Name) > 100 || len(download.Version) > 100 || !webURL(download.URL, false) { + fail(w, 400, "INVALID_INPUT") + return + } + } + value = d + } + result, err := a.saveRecord(kind, id, value, body.Revision) + if errors.Is(err, errConflict) { + fail(w, 409, "CONFLICT") + return + } + if err != nil { + a.internal(w, err) + return + } + a.audit(s, "record_save", kind+"/"+id, "success") + respond(w, 200, result) +} +func (a *App) deleteRecord(w http.ResponseWriter, r *http.Request) { + kind, id := r.PathValue("kind"), r.PathValue("id") + if kind != "announcements" && kind != "mirrors" { + fail(w, 400, "INVALID_INPUT") + return + } + s, ok := a.require(w, r, kind != "announcements") + if !ok { + return + } + var body struct { + Revision int `json:"revision"` + } + if !readJSON(w, r, &body) { + return + } + result, err := a.db.Exec("UPDATE records SET deleted=1,revision=revision+1,updated=? WHERE kind=? AND id=? AND revision=? AND deleted=0", time.Now().UnixMilli(), kind, id, body.Revision) + if err != nil { + a.internal(w, err) + return + } + count, _ := result.RowsAffected() + if count != 1 { + fail(w, 409, "CONFLICT") + return + } + a.audit(s, "record_delete", kind+"/"+id, "success") + respond(w, 200, map[string]bool{"ok": true}) +} +func (a *App) upload(w http.ResponseWriter, r *http.Request) { + s, ok := a.require(w, r, true) + if !ok { + return + } + r.Body = http.MaxBytesReader(w, r.Body, 5<<20) + if err := r.ParseMultipartForm(4 << 20); err != nil { + fail(w, 400, "INVALID_IMAGE") + return + } + if r.MultipartForm != nil { + defer r.MultipartForm.RemoveAll() + } + file, _, err := r.FormFile("file") + if err != nil { + fail(w, 400, "INVALID_IMAGE") + return + } + defer file.Close() + content, err := io.ReadAll(io.LimitReader(file, 4<<20+1)) + if err != nil || len(content) > 4<<20 { + fail(w, 400, "INVALID_IMAGE") + return + } + conf, format, err := image.DecodeConfig(bytes.NewReader(content)) + if err != nil || (format != "png" && format != "jpeg") || conf.Width < 1 || conf.Height < 1 || conf.Width > 8192 || conf.Height > 8192 || int64(conf.Width)*int64(conf.Height) > 32000000 { + fail(w, 400, "INVALID_IMAGE") + return + } + ext := ".png" + if format == "jpeg" { + ext = ".jpg" + } + name := tokenHash(randomToken())[:24] + ext + if err = os.WriteFile(filepath.Join(a.cfg.Data, "uploads", name), content, 0644); err != nil { + a.internal(w, err) + return + } + a.audit(s, "upload", name, "success") + respond(w, 201, map[string]string{"url": "/uploads/" + name}) +} +func (a *App) public(w http.ResponseWriter, r *http.Request) { + site, err := a.record("settings", "site") + if err != nil { + a.internal(w, err) + return + } + metadata, err := a.records("mirrors") + if err != nil { + a.internal(w, err) + return + } + metas := map[string]MirrorMetadata{} + for _, record := range metadata { + var m MirrorMetadata + if decodeData(record.Data, &m) == nil { + metas[record.ID] = m + } + } + jobs, updated, upstreamError := a.cachedJobs() + mirrors := []PublicMirror{} + for _, job := range jobs { + meta, exists := metas[job.Name] + if exists && !meta.Visible { + continue + } + if !exists { + meta = MirrorMetadata{Name: job.Name, Category: "other", Visible: true, Description: Localized{}} + } + mirrors = append(mirrors, publicMirror(job, meta)) + } + sort.Slice(mirrors, func(i, j int) bool { return mirrors[i].Name < mirrors[j].Name }) + announcements, err := a.records("announcements") + if err != nil { + a.internal(w, err) + return + } + published := []Record{} + for _, record := range announcements { + var d Announcement + if decodeData(record.Data, &d) == nil && d.Published { + record.Data = d + published = append(published, record) + } + } + sort.SliceStable(published, func(i, j int) bool { + a, b := published[i].Data.(Announcement), published[j].Data.(Announcement) + if a.Pinned != b.Pinned { + return a.Pinned + } + return a.Date > b.Date + }) + if len(published) > 100 { + published = published[:100] + } + respond(w, 200, map[string]any{"site": site.Data, "mirrors": mirrors, "announcements": published, "source": map[string]any{"updatedAt": updated, "stale": upstreamError != "" || updated == 0 || time.Now().UnixMilli()-updated > 60000, "code": upstreamError}}) +} +func (a *App) overview(w http.ResponseWriter, r *http.Request) { + if _, ok := a.require(w, r, false); !ok { + return + } + jobs, updated, code := a.cachedJobs() + workers := []Worker{} + err := a.managerGet(r.Context(), "/workers", &workers) + if err != nil && code == "" { + code = "TUNASYNC_UNAVAILABLE" + } + respond(w, 200, map[string]any{"jobs": jobs, "workers": workers, "worker": a.cfg.Worker, "updatedAt": updated, "code": code, "configAvailable": a.cfg.WorkerConfig != "", "logsAvailable": a.cfg.Logs != ""}) +} +func (a *App) auditList(w http.ResponseWriter, r *http.Request) { + if _, ok := a.require(w, r, true); !ok { + return + } + rows, err := a.db.Query("SELECT id,username,action,target,result,created FROM audit ORDER BY id DESC LIMIT 200") + if err != nil { + a.internal(w, err) + return + } + defer rows.Close() + out := []map[string]any{} + for rows.Next() { + var id, created int64 + var username, action, target, result string + if err = rows.Scan(&id, &username, &action, &target, &result, &created); err != nil { + a.internal(w, err) + return + } + out = append(out, map[string]any{"id": id, "username": username, "action": action, "target": target, "result": result, "created": created}) + } + respond(w, 200, out) +} +func (a *App) users(w http.ResponseWriter, r *http.Request) { + if _, ok := a.require(w, r, true); !ok { + return + } + rows, err := a.db.Query("SELECT id,username,role,created FROM users ORDER BY id") + if err != nil { + a.internal(w, err) + return + } + defer rows.Close() + out := []map[string]any{} + for rows.Next() { + var id, created int64 + var username, role string + if err = rows.Scan(&id, &username, &role, &created); err != nil { + a.internal(w, err) + return + } + out = append(out, map[string]any{"id": id, "username": username, "role": role, "created": created}) + } + respond(w, 200, out) +} +func (a *App) createUser(w http.ResponseWriter, r *http.Request) { + s, ok := a.require(w, r, true) + if !ok { + return + } + var body struct { + Username string `json:"username"` + Password string `json:"password"` + Role string `json:"role"` + } + if !readJSON(w, r, &body) { + return + } + if !validName(body.Username) || (body.Role != "admin" && body.Role != "editor") { + fail(w, 400, "INVALID_INPUT") + return + } + hash, err := hashPassword(body.Password) + if err != nil { + fail(w, 400, "INVALID_PASSWORD") + return + } + if _, err = a.db.Exec("INSERT INTO users(username,password,role,created) VALUES(?,?,?,?)", body.Username, hash, body.Role, time.Now().UnixMilli()); err != nil { + fail(w, 409, "CONFLICT") + return + } + a.audit(s, "user_create", body.Username, "success") + respond(w, 201, map[string]bool{"ok": true}) +} diff --git a/backend/content_migration.go b/backend/content_migration.go new file mode 100644 index 0000000..1136fc6 --- /dev/null +++ b/backend/content_migration.go @@ -0,0 +1,109 @@ +package main + +import ( + "database/sql" + "encoding/json" + "sort" + "strings" + "time" +) + +// Content is a single shared Chinese edition; interface labels remain localized. +// Keep every original row before converting legacy language maps, including deleted rows. +func migrateSharedContent(db *sql.DB) error { + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS shared_content_backups(kind TEXT NOT NULL,id TEXT NOT NULL,revision INTEGER NOT NULL,data TEXT NOT NULL,created INTEGER NOT NULL,PRIMARY KEY(kind,id,revision))`); err != nil { + return err + } + rows, err := tx.Query("SELECT kind,id,data,revision FROM records WHERE kind IN ('announcements','mirrors')") + if err != nil { + return err + } + type legacyRow struct { + kind, id, before, after string + revision int + } + changes := []legacyRow{} + for rows.Next() { + var row legacyRow + if err = rows.Scan(&row.kind, &row.id, &row.before, &row.revision); err != nil { + rows.Close() + return err + } + var data map[string]json.RawMessage + if err = json.Unmarshal([]byte(row.before), &data); err != nil { + rows.Close() + return err + } + fields := []string{"help"} + if row.kind == "announcements" { + fields = []string{"title", "body"} + } + changed := false + for _, field := range fields { + value, exists := data[field] + if !exists { + data[field] = json.RawMessage(`""`) + changed = true + continue + } + var single string + if string(value) != "null" && json.Unmarshal(value, &single) == nil { + continue + } + var localized map[string]string + if err = json.Unmarshal(value, &localized); err != nil { + rows.Close() + return err + } + // A present Chinese edition wins, including an intentionally empty one. + single, exists = localized["zh-CN"] + if !exists { + single, exists = localized["zh"] + } + if !exists { + keys := make([]string, 0, len(localized)) + for key := range localized { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if strings.TrimSpace(localized[key]) != "" { + single = localized[key] + break + } + } + } + data[field], _ = json.Marshal(single) + changed = true + } + if changed { + raw, e := json.Marshal(data) + if e != nil { + rows.Close() + return e + } + row.after = string(raw) + changes = append(changes, row) + } + } + if err = rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + for _, row := range changes { + now := time.Now().UnixMilli() + if _, err = tx.Exec("INSERT INTO shared_content_backups(kind,id,revision,data,created) VALUES(?,?,?,?,?)", row.kind, row.id, row.revision, row.before, now); err != nil { + return err + } + if _, err = tx.Exec("UPDATE records SET data=?,revision=revision+1,updated=? WHERE kind=? AND id=? AND revision=?", row.after, now, row.kind, row.id, row.revision); err != nil { + return err + } + } + return tx.Commit() +} diff --git a/backend/content_migration_test.go b/backend/content_migration_test.go new file mode 100644 index 0000000..30c94ca --- /dev/null +++ b/backend/content_migration_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestSharedChineseContentMigration(t *testing.T) { + c := fixture(t) + legacy := map[string]any{"title": Localized{"zh-CN": "中文公告", "en-US": "English title"}, "body": Localized{"zh-CN": "# 中文正文", "en-US": "English body"}, "published": true, "pinned": false, "date": time.Now().UTC().Format(time.RFC3339)} + if _, err := c.app.saveRecord("announcements", "legacy", legacy, 0); err != nil { + t.Fatal(err) + } + metadata := map[string]any{"name": "alpine", "description": Localized{"zh-CN": "介绍", "en-US": "Description"}, "help": Localized{"zh-CN": "# 中文帮助", "en-US": "English help"}, "category": "os", "visible": true, "website": ""} + if _, err := c.app.saveRecord("mirrors", "alpine", metadata, 0); err != nil { + t.Fatal(err) + } + if err := migrateSharedContent(c.app.db); err != nil { + t.Fatal(err) + } + record, _ := c.app.record("announcements", "legacy") + var article Announcement + if err := decodeData(record.Data, &article); err != nil { + t.Fatal(err) + } + if article.Title != "中文公告" || article.Body != "# 中文正文" || record.Revision != 2 { + t.Fatal("Chinese edition not preserved") + } + record, _ = c.app.record("mirrors", "alpine") + var mirror MirrorMetadata + if err := decodeData(record.Data, &mirror); err != nil { + t.Fatal(err) + } + if mirror.Help != "# 中文帮助" || mirror.Description["en-US"] != "Description" { + t.Fatal("help migration changed unrelated metadata") + } + var backup string + if err := c.app.db.QueryRow("SELECT data FROM shared_content_backups WHERE kind='announcements' AND id='legacy'").Scan(&backup); err != nil || !strings.Contains(backup, "English body") { + t.Fatal("original language editions were not backed up") + } + if err := migrateSharedContent(c.app.db); err != nil { + t.Fatal(err) + } + record, _ = c.app.record("announcements", "legacy") + if record.Revision != 2 { + t.Fatal("migration is not idempotent") + } + response := c.call("GET", "/api/v1/public", nil, false) + var payload struct{ Announcements []struct{ Data Announcement } } + if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil || len(payload.Announcements) != 1 || payload.Announcements[0].Data.Title != "中文公告" { + t.Fatal("public API did not expose shared strings") + } + c.login("admin") + status(t, c.call("PUT", "/api/v1/admin/records/announcements/legacy", map[string]any{"revision": 2, "data": legacy}, true), 400) + article.Body = " " + status(t, c.call("PUT", "/api/v1/admin/records/announcements/legacy", map[string]any{"revision": 2, "data": article}, true), 400) +} + +func TestSharedContentMigrationKeepsEmptyChineseAndLegacyOnlyContent(t *testing.T) { + c := fixture(t) + for name, help := range map[string]any{"empty-zh": Localized{"zh-CN": "", "en-US": "Alternate text"}, "legacy-only": Localized{"en-US": "Existing content"}, "missing": nil} { + if _, err := c.app.saveRecord("mirrors", name, map[string]any{"name": name, "help": help}, 0); err != nil { + t.Fatal(err) + } + } + if err := migrateSharedContent(c.app.db); err != nil { + t.Fatal(err) + } + for name, want := range map[string]string{"empty-zh": "", "legacy-only": "Existing content", "missing": ""} { + record, _ := c.app.record("mirrors", name) + var mirror MirrorMetadata + if err := decodeData(record.Data, &mirror); err != nil || mirror.Help != want { + t.Fatalf("unexpected migrated help for %s", name) + } + } +} diff --git a/backend/go.mod b/backend/go.mod new file mode 100644 index 0000000..47c18ce --- /dev/null +++ b/backend/go.mod @@ -0,0 +1,20 @@ +module mirrorouc/backend + +go 1.27.0 + +require ( + github.com/BurntSushi/toml v1.6.0 + modernc.org/sqlite v1.61.0 +) + +require ( + github.com/dustin/go-humanize v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.1.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.49.0 // indirect + modernc.org/libc v1.79.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) diff --git a/backend/go.sum b/backend/go.sum new file mode 100644 index 0000000..c86fe05 --- /dev/null +++ b/backend/go.sum @@ -0,0 +1,52 @@ +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= +github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= +modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys= +modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4= +modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o= +modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8= +modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY= +modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/backend/helper_test.go b/backend/helper_test.go new file mode 100644 index 0000000..91d0a3d --- /dev/null +++ b/backend/helper_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "encoding/json" + "net" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +func TestConfigurationHelperTransport(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Unix domain socket integration runs on Linux") + } + c := fixture(t) + c.login("admin") + root, err := os.MkdirTemp("", "mouc-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + socket := filepath.Join(root, "helper.sock") + listener, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + c.app.cfg.ConfigSocket = socket + config, _ := c.app.loadConfig() + numbers := config.global + numbers.Concurrent++ + response := c.call("POST", "/api/v1/admin/config/preview", map[string]any{"baseHash": config.hash, "global": numbers}, true) + status(t, response, 201) + var draft configDraft + json.Unmarshal(response.Body.Bytes(), &draft) + received := make(chan configDraft, 1) + go func() { + connection, e := listener.Accept() + if e != nil { + return + } + defer connection.Close() + connection.SetDeadline(time.Now().Add(5 * time.Second)) + var value configDraft + if json.NewDecoder(connection).Decode(&value) == nil { + received <- value + } + json.NewEncoder(connection).Encode(map[string]string{"code": ""}) + }() + status(t, c.call("POST", "/api/v1/admin/config/apply", map[string]string{"id": draft.ID}, true), 202) + select { + case value := <-received: + if value.ID != draft.ID || value.BaseHash != config.hash || len(value.Files) != 1 { + t.Fatal("helper did not receive the validated draft") + } + case <-time.After(time.Second): + t.Fatal("helper request missing") + } + version, _ := c.app.readVersion(draft.ID) + if version.Status != "applied" { + t.Fatal("helper success not recorded") + } + // Only the helper writes in this mode; the ordinary web process leaves files untouched. + after, _ := c.app.loadConfig() + if after.hash != config.hash { + t.Fatal("web process bypassed configuration helper") + } +} diff --git a/backend/main.go b/backend/main.go new file mode 100644 index 0000000..d419ac9 --- /dev/null +++ b/backend/main.go @@ -0,0 +1,223 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "flag" + "io" + "log" + "net/http" + "os" + "os/signal" + "path/filepath" + "strings" + "sync" + "syscall" + "time" +) + +type Config struct { + Listen, Data, Static, Manager, Worker, WorkerConfig, Logs, Metrics, ConfigSocket string + SecureCookie bool +} +type loginAttempt struct { + at time.Time + count int +} +type App struct { + cfg Config + db *sql.DB + logger *log.Logger + client *http.Client + dummyPassword string + loginMu sync.Mutex + attempts map[string]loginAttempt + jobsMu sync.RWMutex + jobs []Job + jobsUpdated int64 + jobsError string + configMu sync.Mutex +} + +func respond(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(status) + json.NewEncoder(w).Encode(value) +} +func fail(w http.ResponseWriter, status int, code string) { + respond(w, status, map[string]string{"code": code}) +} +func readJSON(w http.ResponseWriter, r *http.Request, target any) bool { + r.Body = http.MaxBytesReader(w, r.Body, 2<<20) + d := json.NewDecoder(r.Body) + d.DisallowUnknownFields() + if err := d.Decode(target); err != nil { + fail(w, 400, "INVALID_INPUT") + return false + } + if err := d.Decode(new(any)); err != io.EOF { + fail(w, 400, "INVALID_INPUT") + return false + } + return true +} +func (a *App) internal(w http.ResponseWriter, err error) { + a.logger.Printf("request failed: %v", err) + fail(w, 500, "INTERNAL_ERROR") +} +func env(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} +func newApp(cfg Config) (*App, error) { + db, err := openDatabase(filepath.Join(cfg.Data, "mirrorouc.db")) + if err != nil { + return nil, err + } + dummy, _ := hashPassword(randomToken()) + a := &App{cfg: cfg, db: db, logger: log.Default(), client: &http.Client{Timeout: 10 * time.Second}, dummyPassword: dummy, attempts: map[string]loginAttempt{}, jobs: []Job{}} + if _, err = a.record("settings", "site"); errors.Is(err, errMissing) { + if _, err = a.saveRecord("settings", "site", initialSettings(), 0); err != nil { + db.Close() + return nil, err + } + } + if err = os.MkdirAll(filepath.Join(cfg.Data, "uploads"), 0755); err != nil { + db.Close() + return nil, err + } + return a, nil +} +func (a *App) handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { + if err := a.db.PingContext(r.Context()); err != nil { + fail(w, 503, "DATABASE_UNAVAILABLE") + return + } + respond(w, 200, map[string]string{"status": "ok"}) + }) + mux.HandleFunc("GET /api/v1/public", a.public) + mux.HandleFunc("GET /api/v1/monitoring", a.monitoring) + mux.HandleFunc("GET /api/v1/job-speeds", a.jobSpeeds) + mux.HandleFunc("POST /api/v1/auth/login", a.login) + mux.HandleFunc("GET /api/v1/auth/session", a.session) + mux.HandleFunc("POST /api/v1/auth/logout", a.logout) + mux.HandleFunc("PUT /api/v1/auth/password", a.password) + mux.HandleFunc("GET /api/v1/admin/overview", a.overview) + mux.HandleFunc("GET /api/v1/admin/audit", a.auditList) + mux.HandleFunc("GET /api/v1/admin/users", a.users) + mux.HandleFunc("POST /api/v1/admin/users", a.createUser) + mux.HandleFunc("GET /api/v1/admin/records/{kind}", a.listRecords) + mux.HandleFunc("PUT /api/v1/admin/records/{kind}/{id}", a.putRecord) + mux.HandleFunc("DELETE /api/v1/admin/records/{kind}/{id}", a.deleteRecord) + mux.HandleFunc("POST /api/v1/admin/uploads", a.upload) + mux.HandleFunc("POST /api/v1/admin/jobs/{name}/{action}", a.command) + mux.HandleFunc("GET /api/v1/admin/jobs/{name}/logs", a.logs) + mux.HandleFunc("GET /api/v1/admin/config", a.configView) + mux.HandleFunc("POST /api/v1/admin/config/preview", a.configPreview) + mux.HandleFunc("POST /api/v1/admin/config/apply", a.configApply) + mux.HandleFunc("GET /api/v1/admin/config/versions", a.configVersions) + mux.HandleFunc("POST /api/v1/admin/config/rollback", a.configRollback) + mux.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) { fail(w, 404, "NOT_FOUND") }) + mux.HandleFunc("GET /uploads/{name}", func(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if !validName(name) || (!strings.HasSuffix(name, ".png") && !strings.HasSuffix(name, ".jpg")) { + http.NotFound(w, r) + return + } + http.ServeFile(w, r, filepath.Join(a.cfg.Data, "uploads", name)) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" && r.Method != "HEAD" { + w.Header().Set("Allow", "GET, HEAD") + fail(w, 405, "METHOD_NOT_ALLOWED") + return + } + path := r.URL.Path + if path == "/" || path == "/status" || path == "/status/" || path == "/admin" || strings.HasPrefix(path, "/admin/") { + http.ServeFile(w, r, filepath.Join(a.cfg.Static, "index.html")) + return + } + if strings.HasPrefix(path, "/assets/") { + http.FileServer(http.Dir(a.cfg.Static)).ServeHTTP(w, r) + return + } + http.NotFound(w, r) + }) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if v := recover(); v != nil { + a.logger.Printf("panic: %v", v) + fail(w, 500, "INTERNAL_ERROR") + } + }() + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("X-Frame-Options", "DENY") + w.Header().Set("Referrer-Policy", "same-origin") + w.Header().Set("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https:; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'") + mux.ServeHTTP(w, r) + }) +} +func main() { + var cfg Config + var initUser string + flag.StringVar(&cfg.Listen, "listen", env("MIRROROUC_LISTEN", "127.0.0.1:8081"), "HTTP listen address") + flag.StringVar(&cfg.Data, "data", env("MIRROROUC_DATA", "./data"), "local data directory") + flag.StringVar(&cfg.Static, "static", env("MIRROROUC_STATIC", "../dist"), "frontend dist directory") + flag.StringVar(&cfg.Manager, "manager", env("MIRROROUC_MANAGER", "http://127.0.0.1:12345"), "tunasync Manager URL") + flag.StringVar(&cfg.Worker, "worker", env("MIRROROUC_WORKER", ""), "worker ID") + flag.StringVar(&cfg.WorkerConfig, "worker-config", env("MIRROROUC_WORKER_CONFIG", ""), "worker TOML configuration") + flag.StringVar(&cfg.Logs, "logs", env("MIRROROUC_LOGS", ""), "tunasync log root") + flag.StringVar(&cfg.Metrics, "metrics", env("MIRROROUC_METRICS", ""), "metrics history JSON") + flag.StringVar(&cfg.ConfigSocket, "config-socket", env("MIRROROUC_CONFIG_SOCKET", ""), "restricted configuration helper Unix socket") + flag.BoolVar(&cfg.SecureCookie, "secure-cookies", os.Getenv("MIRROROUC_SECURE_COOKIES") == "true", "require HTTPS cookies") + flag.StringVar(&initUser, "init-admin", "", "create initial administrator using MIRROROUC_ADMIN_PASSWORD (generated if absent)") + flag.Parse() + a, err := newApp(cfg) + if err != nil { + log.Fatal(err) + } + defer a.db.Close() + if initUser != "" { + password := os.Getenv("MIRROROUC_ADMIN_PASSWORD") + generated := password == "" + if generated { + password = randomToken() + } + hash, err := hashPassword(password) + if err != nil { + log.Fatal(err) + } + if !validName(initUser) { + log.Fatal("invalid username") + } + if _, err = a.db.Exec("INSERT INTO users(username,password,role,created) VALUES(?,?,'admin',?)", initUser, hash, time.Now().UnixMilli()); err != nil { + log.Fatal(err) + } + if generated { + os.Stdout.WriteString(password + "\n") + } + return + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + a.loadCache() + go a.poll(ctx) + srv := &http.Server{Addr: cfg.Listen, Handler: a.handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 20 * time.Second, WriteTimeout: 30 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 1 << 20} + go func() { + <-ctx.Done() + stop, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + srv.Shutdown(stop) + }() + log.Printf("MirrorOUC API listening on %s", cfg.Listen) + if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatal(err) + } +} diff --git a/backend/models.go b/backend/models.go new file mode 100644 index 0000000..70705b6 --- /dev/null +++ b/backend/models.go @@ -0,0 +1,127 @@ +package main + +import "time" + +type Localized map[string]string +type Link struct { + Title Localized `json:"title"` + URL string `json:"url"` +} +type RelatedLink struct { + Title Localized `json:"title"` + Kind string `json:"kind"` + URL string `json:"url"` +} +type Download struct { + Name string `json:"name"` + Version string `json:"version"` + URL string `json:"url"` +} +type Settings struct { + Name Localized `json:"name"` + Description Localized `json:"description"` + LogoLight string `json:"logoLight"` + LogoDark string `json:"logoDark"` + Favicon string `json:"favicon"` + Avatar string `json:"avatar"` + Email string `json:"email"` + Address Localized `json:"address"` + Links []Link `json:"links"` + RelatedLinks []RelatedLink `json:"relatedLinks"` + Downloads []Download `json:"downloads"` +} +type Announcement struct { + Title string `json:"title"` + Body string `json:"body"` + Published bool `json:"published"` + Pinned bool `json:"pinned"` + Date string `json:"date"` +} +type MirrorMetadata struct { + Name string `json:"name"` + Description Localized `json:"description"` + Help string `json:"help"` + Category string `json:"category"` + Visible bool `json:"visible"` + Website string `json:"website"` +} +type Job struct { + Name string `json:"name"` + Worker string `json:"worker"` + Status string `json:"status"` + LastUpdate time.Time `json:"last_update"` + LastStarted time.Time `json:"last_started"` + LastEnded time.Time `json:"last_ended"` + NextSchedule time.Time `json:"next_schedule"` + Upstream string `json:"upstream"` + Size string `json:"size"` + Error string `json:"error_msg"` +} +type PublicMirror struct { + Name string `json:"name"` + Description Localized `json:"description"` + Help string `json:"help"` + Category string `json:"category"` + Status string `json:"status"` + LastSuccess *int64 `json:"lastSuccess"` + LastAttempt *int64 `json:"lastAttempt"` + SyncStarted *int64 `json:"syncStarted"` + NextSync *int64 `json:"nextSync"` + Size *float64 `json:"size"` + Website string `json:"website"` + Upstream string `json:"upstream"` + BrowseURL string `json:"browseUrl"` +} +type Worker struct { + ID string `json:"id"` + LastOnline time.Time `json:"last_online"` +} +type Record struct { + ID string `json:"id"` + Data any `json:"data"` + Revision int `json:"revision"` + UpdatedAt int64 `json:"updatedAt"` +} +type Session struct { + UserID int64 + Username string + Role string + CSRF string +} + +func initialSettings() Settings { + return Settings{ + Name: Localized{"zh-CN": "中国海洋大学软件镜像源", "en-US": "Ocean University of China Software Mirrors"}, + Description: Localized{"zh-CN": "中国海洋大学软件镜像源(MirrorOUC)由中国海洋大学爱特工作室承建。", "en-US": "MirrorOUC is built by IT Studio, Ocean University of China."}, + LogoLight: "/assets/mirrorOUC_dark.png", LogoDark: "/assets/mirrorOUC.png", Favicon: "/assets/ouc-seal.png", Avatar: "/assets/ouc-seal.png", Email: "contact@itstudio.club", + Address: Localized{"zh-CN": "山东省青岛市中国海洋大学西海岸校区 · 信息南楼五八创新创业工坊", "en-US": "58 Innovation Workshop, Information South Building, West Coast Campus, Ocean University of China, Qingdao, Shandong"}, + RelatedLinks: initialRelatedLinks(), + Links: []Link{ + {Localized{"zh-CN": "中国海洋大学", "en-US": "Ocean University of China"}, "https://www.ouc.edu.cn/"}, + {Localized{"zh-CN": "爱特工作室", "en-US": "IT Studio"}, "https://www.itouc.cn/"}, + {Localized{"zh-CN": "信息科学与工程学部", "en-US": "Faculty of Information Science and Engineering"}, "https://it.ouc.edu.cn/"}, + {Localized{"zh-CN": "爱特荣誉墙", "en-US": "IT Studio honors"}, "https://honor.itouc.cn/"}, + {Localized{"zh-CN": "GitOUC", "en-US": "GitOUC"}, "https://git.itouc.cn/"}, + {Localized{"zh-CN": "爱特博客网", "en-US": "IT Studio blog"}, "https://blog.itouc.cn/"}, + }, Downloads: []Download{}, + } +} + +func initialRelatedLinks() []RelatedLink { + return []RelatedLink{ + {Title: Localized{"zh-CN": "软件源使用帮助", "en-US": "Mirror usage guides"}, Kind: "help", URL: ""}, + {Title: Localized{"zh-CN": "中国海洋大学爱特工作室", "en-US": "IT Studio, Ocean University of China"}, Kind: "url", URL: "https://www.itouc.cn/"}, + {Title: Localized{"zh-CN": "中国海洋大学", "en-US": "Ocean University of China"}, Kind: "url", URL: "https://www.ouc.edu.cn/"}, + } +} + +// Older settings records get the existing sidebar entries until explicitly edited. +// An empty array is intentional and must stay empty. +func settingsDefaults(data any) any { + if object, ok := data.(map[string]any); ok { + if value, exists := object["relatedLinks"]; !exists || value == nil { + object["relatedLinks"] = initialRelatedLinks() + } + } + return data +} diff --git a/backend/monitoring.go b/backend/monitoring.go new file mode 100644 index 0000000..690798f --- /dev/null +++ b/backend/monitoring.go @@ -0,0 +1,147 @@ +package main + +import ( + "encoding/json" + "math" + "net/http" + "os" + "strconv" + "time" +) + +type metricsHistory struct { + UpdatedAt int64 `json:"updatedAt"` + MemoryTotal float64 `json:"memoryTotal"` + Disks []map[string]any `json:"disks"` + Samples []map[string]float64 `json:"samples"` +} +type metricStats struct { + Mean float64 `json:"mean"` + Latest float64 `json:"latest"` + Max float64 `json:"max"` +} +type metricSeries struct { + ID string `json:"id"` + LabelKey string `json:"labelKey"` + Color string `json:"color"` + Values []float64 `json:"values"` + Stats metricStats `json:"stats"` +} +type metricPanel struct { + ID string `json:"id"` + TitleKey string `json:"titleKey"` + Unit string `json:"unit"` + Maximum float64 `json:"maximum"` + Stacked bool `json:"stacked"` + Available bool `json:"available"` + Series []metricSeries `json:"series"` +} + +func niceMaximum(value float64) float64 { + if value <= 0 { + return 1 + } + power := math.Pow(10, math.Floor(math.Log10(value))) + for _, step := range []float64{1, 2, 5, 10} { + if step*power >= value*1.05 { + return step * power + } + } + return 20 * power +} +func (a *App) monitoring(w http.ResponseWriter, r *http.Request) { + hours, err := strconv.Atoi(r.URL.Query().Get("hours")) + if err != nil || (hours != 1 && hours != 6 && hours != 24) { + fail(w, 400, "INVALID_RANGE") + return + } + var history metricsHistory + if a.cfg.Metrics == "" { + fail(w, 503, "METRICS_UNAVAILABLE") + return + } + raw, err := os.ReadFile(a.cfg.Metrics) + if err != nil || len(raw) > 32<<20 { + fail(w, 503, "METRICS_UNAVAILABLE") + return + } + if err = json.Unmarshal(raw, &history); err != nil { + fail(w, 503, "METRICS_UNAVAILABLE") + return + } + samples := []map[string]float64{} + cutoff := float64(time.Now().Add(-time.Duration(hours) * time.Hour).UnixMilli()) + for _, sample := range history.Samples { + if sample["time"] >= cutoff { + samples = append(samples, sample) + } + } + indices := []int{} + if len(samples) <= 241 { + for i := range samples { + indices = append(indices, i) + } + } else { + for i := 0; i < 241; i++ { + indices = append(indices, int(math.Round(float64(i)*float64(len(samples)-1)/240))) + } + } + timestamps := []int64{} + for _, index := range indices { + timestamps = append(timestamps, int64(samples[index]["time"])) + } + makeSeries := func(id, label, color string) metricSeries { + series := metricSeries{ID: id, LabelKey: "monitor.series." + label, Color: "var(--" + color + ")", Values: []float64{}} + count := 0 + for _, sample := range samples { + if value, ok := sample[id]; ok { + series.Stats.Mean += value + series.Stats.Max = math.Max(value, series.Stats.Max) + series.Stats.Latest = value + count++ + } + } + if count > 0 { + series.Stats.Mean /= float64(count) + } + for _, index := range indices { + series.Values = append(series.Values, samples[index][id]) + } + return series + } + panel := func(id, unit string, maximum float64, stacked bool, series ...metricSeries) metricPanel { + available := len(samples) > 1 + peak := float64(0) + for _, sample := range samples { + sum := float64(0) + for _, s := range series { + value, ok := sample[s.ID] + if !ok { + available = false + } + if stacked { + sum += value + } else { + sum = math.Max(sum, value) + } + } + peak = math.Max(peak, sum) + } + if maximum == 0 { + maximum = niceMaximum(peak) + } + if maximum <= 0 { + maximum = 1 + } + return metricPanel{id, "monitor.charts." + id, unit, maximum, stacked, available, series} + } + panels := []metricPanel{ + panel("ipv4", "gigabits", 0, true, makeSeries("https4", "https", "blue"), makeSeries("rsync4", "rsync", "violet")), + panel("ipv6", "gigabits", 0, true, makeSeries("https6", "https", "blue"), makeSeries("rsync6", "rsync", "violet")), + panel("iops", "operations", 0, false, makeSeries("readOps", "read", "green"), makeSeries("writeOps", "write", "amber")), + panel("throughput", "gigabytes", 0, false, makeSeries("readBytes", "read", "blue"), makeSeries("writeBytes", "write", "violet")), + panel("cpu", "percent", 100, true, makeSeries("user", "user", "blue"), makeSeries("system", "system", "green"), makeSeries("wait", "wait", "amber"), makeSeries("idle", "idle", "chart-muted")), + panel("memory", "memory", history.MemoryTotal, true, makeSeries("used", "used", "blue"), makeSeries("cached", "cached", "green"), makeSeries("buffers", "buffers", "amber"), makeSeries("free", "free", "chart-muted")), + } + respond(w, 200, map[string]any{"timestamps": timestamps, "panels": panels, "disks": history.Disks, "updatedAt": history.UpdatedAt, "stale": time.Now().UnixMilli()-history.UpdatedAt > 90000}) +} diff --git a/backend/related_links_test.go b/backend/related_links_test.go new file mode 100644 index 0000000..de5ffac --- /dev/null +++ b/backend/related_links_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "encoding/json" + "path/filepath" + "reflect" + "testing" +) + +func TestRelatedLinksPersistIndependentlyAndSupportOlderForms(t *testing.T) { + c := fixture(t) + c.login("admin") + initial, _ := c.app.record("settings", "site") + var legacy map[string]any + decodeData(initial.Data, &legacy) + delete(legacy, "relatedLinks") + if _, err := c.app.saveRecord("settings", "site", legacy, initial.Revision); err != nil { + t.Fatal(err) + } + current, _ := c.app.record("settings", "site") + var settings Settings + decodeData(current.Data, &settings) + if len(settings.RelatedLinks) != 3 || settings.RelatedLinks[0].Kind != "help" { + t.Fatal("legacy sidebar defaults missing") + } + response := c.call("GET", "/api/v1/admin/records/settings", nil, false) + var adminRecords []struct{ Data Settings } + json.Unmarshal(response.Body.Bytes(), &adminRecords) + if len(adminRecords) != 1 || !reflect.DeepEqual(adminRecords[0].Data.RelatedLinks, settings.RelatedLinks) { + t.Fatal("admin/public defaults differ") + } + footer := append([]Link{}, settings.Links...) + settings.RelatedLinks = []RelatedLink{ + {Title: Localized{"zh-CN": "站点状态", "en-US": "Status"}, Kind: "url", URL: "/status"}, + {Title: Localized{"zh-CN": "使用帮助"}, Kind: "help", URL: ""}, + } + response = c.call("PUT", "/api/v1/admin/records/settings/site", map[string]any{"data": settings, "revision": current.Revision}, true) + status(t, response, 200) + var saved Record + json.Unmarshal(response.Body.Bytes(), &saved) + legacy = nil + decodeData(saved.Data, &legacy) + delete(legacy, "relatedLinks") + response = c.call("PUT", "/api/v1/admin/records/settings/site", map[string]any{"data": legacy, "revision": saved.Revision}, true) + status(t, response, 200) + json.Unmarshal(response.Body.Bytes(), &saved) + response = c.call("GET", "/api/v1/public", nil, false) + var public struct{ Site Settings } + json.Unmarshal(response.Body.Bytes(), &public) + if !reflect.DeepEqual(public.Site.RelatedLinks, settings.RelatedLinks) || !reflect.DeepEqual(public.Site.Links, footer) { + t.Fatal("sidebar edits changed footer links or were lost by older form") + } + settings.RelatedLinks = []RelatedLink{} + status(t, c.call("PUT", "/api/v1/admin/records/settings/site", map[string]any{"data": settings, "revision": saved.Revision}, true), 200) + c.app.db.Close() + var err error + c.app.db, err = openDatabase(filepath.Join(c.app.cfg.Data, "mirrorouc.db")) + if err != nil { + t.Fatal(err) + } + response = c.call("GET", "/api/v1/public", nil, false) + public.Site = Settings{} + json.Unmarshal(response.Body.Bytes(), &public) + if public.Site.RelatedLinks == nil || len(public.Site.RelatedLinks) != 0 { + t.Fatal("explicitly cleared sidebar reset after restart") + } +} + +func TestRelatedLinkValidation(t *testing.T) { + c := fixture(t) + c.login("admin") + record, _ := c.app.record("settings", "site") + settings := initialSettings() + bad := []RelatedLink{ + {Title: Localized{"zh-CN": "测试"}, Kind: "url", URL: "javascript:alert(1)"}, + {Title: Localized{"zh-CN": "测试"}, Kind: "url", URL: "data:text/html,example"}, + {Title: Localized{"zh-CN": "测试"}, Kind: "url", URL: "//example.invalid"}, + {Title: Localized{"zh-CN": "测试"}, Kind: "url", URL: "/\\example.invalid"}, + {Title: Localized{"zh-CN": "测试"}, Kind: "shell", URL: "/status"}, + {Title: Localized{"zh-CN": "测试"}, Kind: "help", URL: "https://example.invalid/"}, + {Title: Localized{"zh-CN": " "}, Kind: "url", URL: "https://example.invalid/"}, + } + for _, link := range bad { + settings.RelatedLinks = []RelatedLink{link} + status(t, c.call("PUT", "/api/v1/admin/records/settings/site", map[string]any{"data": settings, "revision": record.Revision}, true), 400) + } + for _, value := range []string{"https://example.invalid/", "http://example.invalid/", "/status", "/status?range=1#monitor-sync"} { + if !navigationURL(value) { + t.Fatalf("valid navigation URL rejected: %s", value) + } + } + c.login("editor") + status(t, c.call("GET", "/api/v1/admin/records/settings", nil, false), 403) +} diff --git a/backend/security_test.go b/backend/security_test.go new file mode 100644 index 0000000..8e6d2bc --- /dev/null +++ b/backend/security_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "bytes" + "encoding/json" + "image" + "image/png" + "mime/multipart" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +func TestUploadsAreValidatedAndNotListed(t *testing.T) { + c := fixture(t) + c.login("admin") + upload := func(value []byte) *httptest.ResponseRecorder { + var body bytes.Buffer + form := multipart.NewWriter(&body) + part, _ := form.CreateFormFile("file", "../../unsafe.html") + part.Write(value) + form.Close() + request := httptest.NewRequest("POST", "http://localhost/api/v1/admin/uploads", &body) + request.Header.Set("Content-Type", form.FormDataContentType()) + request.Header.Set("X-CSRF-Token", c.csrf) + request.AddCookie(c.cookie) + response := httptest.NewRecorder() + c.app.handler().ServeHTTP(response, request) + return response + } + status(t, upload([]byte("")), 400) + var pixel bytes.Buffer + png.Encode(&pixel, image.NewRGBA(image.Rect(0, 0, 1, 1))) + result := upload(pixel.Bytes()) + status(t, result, 201) + var value map[string]string + json.Unmarshal(result.Body.Bytes(), &value) + status(t, c.call("GET", value["url"], nil, false), 200) + status(t, c.call("GET", "/uploads/", nil, false), 404) + status(t, c.call("GET", "/uploads/mirrorouc.db", nil, false), 404) +} + +func TestMonitoringUsesRawStatistics(t *testing.T) { + c := fixture(t) + history := metricsHistory{UpdatedAt: time.Now().UnixMilli(), MemoryTotal: 32, Samples: []map[string]float64{}} + for index := 0; index < 500; index++ { + value := float64(1) + if index == 1 { + value = 200 + } + history.Samples = append(history.Samples, map[string]float64{"time": float64(time.Now().Add(time.Duration(index-500) * time.Second).UnixMilli()), "https4": value, "rsync4": 0}) + } + raw, _ := json.Marshal(history) + path := filepath.Join(t.TempDir(), "history.json") + os.WriteFile(path, raw, 0644) + c.app.cfg.Metrics = path + response := c.call("GET", "/api/v1/monitoring?hours=1", nil, false) + status(t, response, 200) + var result struct { + Panels []metricPanel + Timestamps []int64 + } + json.Unmarshal(response.Body.Bytes(), &result) + if len(result.Timestamps) != 241 || result.Panels[0].Series[0].Stats.Max != 200 || !result.Panels[0].Available || result.Panels[1].Available { + t.Fatal("incorrect sampled statistics or missing-metric handling") + } + status(t, c.call("GET", "/api/v1/monitoring?hours=2", nil, false), 400) +} +func TestJobSpeedsFilterInactiveAndStaleData(t *testing.T) { + c := fixture(t) + root := t.TempDir() + c.app.cfg.Metrics = filepath.Join(root, "history.json") + c.app.jobsMu.Lock() + c.app.jobs = []Job{{Name: "debian", Status: "syncing"}, {Name: "ubuntu", Status: "success"}} + c.app.jobsMu.Unlock() + snapshot := speedSnapshot{UpdatedAt: time.Now().UnixMilli(), Interval: 2, Available: true, Jobs: map[string]jobSpeed{"debian": {BytesPerSecond: 12345, Connections: 3}, "ubuntu": {BytesPerSecond: 99999}, "secret-job": {BytesPerSecond: 99999}}} + write := func() { + raw, _ := json.Marshal(snapshot) + os.WriteFile(filepath.Join(root, "job-speeds.json"), raw, 0644) + } + write() + response := c.call("GET", "/api/v1/job-speeds", nil, false) + status(t, response, 200) + var result speedSnapshot + json.Unmarshal(response.Body.Bytes(), &result) + if len(result.Jobs) != 1 || result.Jobs["debian"].BytesPerSecond != 12345 { + t.Fatal("inactive or unknown jobs leaked into speeds") + } + snapshot.UpdatedAt = time.Now().Add(-time.Minute).UnixMilli() + write() + response = c.call("GET", "/api/v1/job-speeds", nil, false) + json.Unmarshal(response.Body.Bytes(), &result) + // Decode into a new map: encoding/json merges map entries when reusing one. + result = speedSnapshot{} + json.Unmarshal(response.Body.Bytes(), &result) + if !result.Stale || len(result.Jobs) != 0 { + t.Fatal("outdated rates presented as current") + } +} diff --git a/backend/speeds.go b/backend/speeds.go new file mode 100644 index 0000000..2dd1e05 --- /dev/null +++ b/backend/speeds.go @@ -0,0 +1,46 @@ +package main + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + "time" +) + +type jobSpeed struct { + BytesPerSecond float64 `json:"bytesPerSecond"` + Connections int `json:"connections"` +} +type speedSnapshot struct { + UpdatedAt int64 `json:"updatedAt"` + Interval float64 `json:"interval"` + Available bool `json:"available"` + Jobs map[string]jobSpeed `json:"jobs"` + Stale bool `json:"stale"` +} + +func (a *App) jobSpeeds(w http.ResponseWriter, r *http.Request) { + if a.cfg.Metrics == "" { + fail(w, 503, "SPEEDS_UNAVAILABLE") + return + } + raw, err := os.ReadFile(filepath.Join(filepath.Dir(a.cfg.Metrics), "job-speeds.json")) + var snapshot speedSnapshot + if err != nil || len(raw) > 1<<20 || json.Unmarshal(raw, &snapshot) != nil { + fail(w, 503, "SPEEDS_UNAVAILABLE") + return + } + snapshot.Stale = time.Now().UnixMilli()-snapshot.UpdatedAt > 15000 || snapshot.UpdatedAt > time.Now().Add(time.Minute).UnixMilli() + filtered := map[string]jobSpeed{} + jobs, _, code := a.cachedJobs() + if code == "" && snapshot.Available && !snapshot.Stale { + for _, job := range jobs { + if speed, exists := snapshot.Jobs[job.Name]; exists && job.Status == "syncing" && speed.BytesPerSecond >= 0 && speed.Connections >= 0 { + filtered[job.Name] = speed + } + } + } + snapshot.Jobs = filtered + respond(w, 200, snapshot) +} diff --git a/backend/store.go b/backend/store.go new file mode 100644 index 0000000..89da6c8 --- /dev/null +++ b/backend/store.go @@ -0,0 +1,117 @@ +package main + +import ( + "database/sql" + "encoding/json" + "errors" + _ "modernc.org/sqlite" + "os" + "path/filepath" + "time" +) + +var errConflict = errors.New("CONFLICT") +var errMissing = errors.New("NOT_FOUND") + +func openDatabase(path string) (*sql.DB, error) { + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", path+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)") + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS users(id INTEGER PRIMARY KEY, username TEXT NOT NULL UNIQUE, password TEXT NOT NULL, role TEXT NOT NULL CHECK(role IN ('admin','editor')), created INTEGER NOT NULL); +CREATE TABLE IF NOT EXISTS sessions(token TEXT PRIMARY KEY, user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, csrf TEXT NOT NULL, expires INTEGER NOT NULL); +CREATE TABLE IF NOT EXISTS records(kind TEXT NOT NULL,id TEXT NOT NULL,data TEXT NOT NULL,revision INTEGER NOT NULL DEFAULT 1,updated INTEGER NOT NULL,deleted INTEGER NOT NULL DEFAULT 0,PRIMARY KEY(kind,id)); +CREATE TABLE IF NOT EXISTS audit(id INTEGER PRIMARY KEY,username TEXT NOT NULL,action TEXT NOT NULL,target TEXT NOT NULL,result TEXT NOT NULL,created INTEGER NOT NULL); +CREATE TABLE IF NOT EXISTS config_versions(id TEXT PRIMARY KEY,files TEXT NOT NULL,base_hash TEXT NOT NULL,status TEXT NOT NULL,created INTEGER NOT NULL,username TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS cache(key TEXT PRIMARY KEY,data TEXT NOT NULL,updated INTEGER NOT NULL);`) + if err != nil { + db.Close() + return nil, err + } + if err = migrateSharedContent(db); err != nil { + db.Close() + return nil, err + } + return db, nil +} +func (a *App) record(kind, id string) (Record, error) { + var r Record + var raw string + r.ID = id + err := a.db.QueryRow("SELECT data,revision,updated FROM records WHERE kind=? AND id=? AND deleted=0", kind, id).Scan(&raw, &r.Revision, &r.UpdatedAt) + if errors.Is(err, sql.ErrNoRows) { + return r, errMissing + } + if err != nil { + return r, err + } + err = json.Unmarshal([]byte(raw), &r.Data) + if kind == "settings" && id == "site" { + r.Data = settingsDefaults(r.Data) + } + return r, err +} +func (a *App) records(kind string) ([]Record, error) { + rows, err := a.db.Query("SELECT id,data,revision,updated FROM records WHERE kind=? AND deleted=0 ORDER BY updated DESC,id", kind) + if err != nil { + return nil, err + } + defer rows.Close() + out := []Record{} + for rows.Next() { + var r Record + var raw string + if err = rows.Scan(&r.ID, &raw, &r.Revision, &r.UpdatedAt); err != nil { + return nil, err + } + if err = json.Unmarshal([]byte(raw), &r.Data); err != nil { + return nil, err + } + if kind == "settings" && r.ID == "site" { + r.Data = settingsDefaults(r.Data) + } + out = append(out, r) + } + return out, rows.Err() +} +func (a *App) saveRecord(kind, id string, data any, revision int) (Record, error) { + raw, err := json.Marshal(data) + if err != nil { + return Record{}, err + } + now := time.Now().UnixMilli() + if revision == 0 { + _, err = a.db.Exec("INSERT INTO records(kind,id,data,revision,updated) VALUES(?,?,?,1,?)", kind, id, string(raw), now) + if err != nil { + return Record{}, errConflict + } + } else { + var result sql.Result + result, err = a.db.Exec("UPDATE records SET data=?,revision=revision+1,updated=?,deleted=0 WHERE kind=? AND id=? AND revision=? AND deleted=0", string(raw), now, kind, id, revision) + if err != nil { + return Record{}, err + } + n, _ := result.RowsAffected() + if n != 1 { + return Record{}, errConflict + } + } + return a.record(kind, id) +} +func (a *App) audit(s Session, action, target, result string) { + _, err := a.db.Exec("INSERT INTO audit(username,action,target,result,created) VALUES(?,?,?,?,?)", s.Username, action, target, result, time.Now().UnixMilli()) + if err != nil { + a.logger.Printf("audit write failed: %v", err) + } +} +func decodeData(value any, target any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + return json.Unmarshal(raw, target) +} diff --git a/backend/tunasync.go b/backend/tunasync.go new file mode 100644 index 0000000..6bc32f6 --- /dev/null +++ b/backend/tunasync.go @@ -0,0 +1,263 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +func (a *App) managerGet(ctx context.Context, path string, target any) error { + req, err := http.NewRequestWithContext(ctx, "GET", strings.TrimRight(a.cfg.Manager, "/")+path, nil) + if err != nil { + return err + } + res, err := a.client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return fmt.Errorf("manager HTTP %d", res.StatusCode) + } + return json.NewDecoder(io.LimitReader(res.Body, 8<<20)).Decode(target) +} +func (a *App) sendCommand(ctx context.Context, name, action string) error { + if a.cfg.Worker == "" { + return errors.New("worker not configured") + } + data, _ := json.Marshal(map[string]any{"worker_id": a.cfg.Worker, "mirror_id": name, "cmd": action, "args": []string{}, "options": map[string]bool{}}) + req, err := http.NewRequestWithContext(ctx, "POST", strings.TrimRight(a.cfg.Manager, "/")+"/cmd", bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + res, err := a.client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + io.Copy(io.Discard, io.LimitReader(res.Body, 1<<20)) + if res.StatusCode < 200 || res.StatusCode >= 300 { + return fmt.Errorf("manager command HTTP %d", res.StatusCode) + } + return nil +} +func (a *App) cachedJobs() ([]Job, int64, string) { + a.jobsMu.RLock() + defer a.jobsMu.RUnlock() + jobs := append([]Job{}, a.jobs...) + return jobs, a.jobsUpdated, a.jobsError +} +func (a *App) loadCache() { + var raw string + var updated int64 + if a.db.QueryRow("SELECT data,updated FROM cache WHERE key='jobs'").Scan(&raw, &updated) == nil { + var jobs []Job + if json.Unmarshal([]byte(raw), &jobs) == nil { + a.jobs = jobs + a.jobsUpdated = updated + a.jobsError = "TUNASYNC_UNAVAILABLE" + } + } +} +func (a *App) refresh(ctx context.Context) { + jobs := []Job{} + err := error(nil) + if a.cfg.Worker == "" { + err = errors.New("worker not configured") + } else { + err = a.managerGet(ctx, "/workers/"+url.PathEscape(a.cfg.Worker)+"/jobs", &jobs) + } + a.jobsMu.Lock() + if err != nil { + a.jobsError = "TUNASYNC_UNAVAILABLE" + a.jobsMu.Unlock() + a.logger.Printf("tunasync poll: %v", err) + return + } + a.jobs = jobs + a.jobsUpdated = time.Now().UnixMilli() + a.jobsError = "" + updated := a.jobsUpdated + a.jobsMu.Unlock() + raw, _ := json.Marshal(jobs) + if _, err = a.db.Exec("INSERT INTO cache(key,data,updated) VALUES('jobs',?,?) ON CONFLICT(key) DO UPDATE SET data=excluded.data,updated=excluded.updated", string(raw), updated); err != nil { + a.logger.Printf("cache: %v", err) + } +} +func (a *App) poll(ctx context.Context) { + a.refresh(ctx) + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + a.refresh(ctx) + } + } +} +func stamp(value time.Time) *int64 { + if value.IsZero() || value.Unix() <= 0 { + return nil + } + ms := value.UnixMilli() + return &ms +} + +var humanSize = regexp.MustCompile(`(?i)^\s*(\d+(?:\.\d+)?)\s*([kmgtpe]?)(?:i?b)?\s*$`) + +func parseSize(value string) *float64 { + parts := humanSize.FindStringSubmatch(value) + if parts == nil { + return nil + } + n, err := strconv.ParseFloat(parts[1], 64) + if err != nil || math.IsNaN(n) || math.IsInf(n, 0) { + return nil + } + power := strings.Index("kmgtpe", strings.ToLower(parts[2])) + 1 + if parts[2] == "" { + power = 0 + } + n *= math.Pow(1024, float64(power)) + return &n +} +func redactURL(value string) string { + u, err := url.Parse(value) + if err != nil { + return "" + } + u.User = nil + u.RawQuery = "" + u.Fragment = "" + return u.String() +} +func publicMirror(job Job, meta MirrorMetadata) PublicMirror { + status := map[string]string{"success": "success", "syncing": "syncing", "pre-syncing": "waiting", "failed": "failed", "paused": "paused", "disabled": "paused", "none": "unknown"}[job.Status] + if status == "" { + status = "unknown" + } + return PublicMirror{Name: job.Name, Description: meta.Description, Help: meta.Help, Category: meta.Category, Status: status, LastSuccess: stamp(job.LastUpdate), LastAttempt: stamp(job.LastStarted), SyncStarted: stamp(job.LastStarted), NextSync: stamp(job.NextSchedule), Size: parseSize(job.Size), Website: meta.Website, Upstream: redactURL(job.Upstream), BrowseURL: "/mirrors/" + url.PathEscape(job.Name) + "/"} +} +func (a *App) command(w http.ResponseWriter, r *http.Request) { + s, ok := a.require(w, r, true) + if !ok { + return + } + name, action := r.PathValue("name"), r.PathValue("action") + if action != "start" && action != "stop" && action != "restart" && action != "disable" { + fail(w, 400, "INVALID_ACTION") + return + } + jobs, _, code := a.cachedJobs() + found := false + for _, job := range jobs { + if job.Name == name { + found = true + break + } + } + if !validName(name) || !found { + fail(w, 404, "NOT_FOUND") + return + } + if code != "" { + fail(w, 503, "TUNASYNC_UNAVAILABLE") + return + } + if err := a.sendCommand(r.Context(), name, action); err != nil { + a.audit(s, "job_"+action, name, "uncertain") + a.logger.Printf("command: %v", err) + fail(w, 502, "COMMAND_UNCERTAIN") + return + } + a.audit(s, "job_"+action, name, "submitted") + a.refresh(r.Context()) + respond(w, 202, map[string]string{"status": "submitted"}) +} +func (a *App) logs(w http.ResponseWriter, r *http.Request) { + if _, ok := a.require(w, r, true); !ok { + return + } + name := r.PathValue("name") + if a.cfg.Logs == "" { + fail(w, 503, "LOGS_UNAVAILABLE") + return + } + if !validName(name) { + fail(w, 400, "INVALID_INPUT") + return + } + base, baseErr := filepath.EvalSymlinks(a.cfg.Logs) + if baseErr != nil { + fail(w, 503, "LOGS_UNAVAILABLE") + return + } + root, err := filepath.EvalSymlinks(filepath.Join(base, name)) + if err != nil { + fail(w, 404, "NOT_FOUND") + return + } + rootRel, err := filepath.Rel(base, root) + if err != nil || rootRel == ".." || strings.HasPrefix(rootRel, ".."+string(os.PathSeparator)) || filepath.IsAbs(rootRel) { + fail(w, 403, "FORBIDDEN") + return + } + path, err := filepath.EvalSymlinks(filepath.Join(root, "latest")) + if err != nil { + fail(w, 404, "NOT_FOUND") + return + } + rel, err := filepath.Rel(root, path) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) || filepath.IsAbs(rel) { + fail(w, 403, "FORBIDDEN") + return + } + f, err := os.Open(path) + if err != nil { + fail(w, 404, "NOT_FOUND") + return + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() { + fail(w, 400, "INVALID_INPUT") + return + } + truncated := info.Size() > 65536 + if truncated { + if _, err = f.Seek(-65536, io.SeekEnd); err != nil { + a.internal(w, err) + return + } + } + content, err := io.ReadAll(io.LimitReader(f, 65536)) + if err != nil { + a.internal(w, err) + return + } + respond(w, 200, map[string]any{"content": strings.ToValidUTF8(string(content), "�"), "truncated": truncated}) +} +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..5717664 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,100 @@ +# MirrorOUC 部署 + +当前预览:`http://100.69.2.2:8080`,后台:`/admin`。这是单 Worker 适配版本,使用服务器现有的 tunasync Manager、Worker 和并行 rsync 配置,不重新安装或替换同步引擎。 + +## 运行结构 + +| 服务 | 位置与用途 | +| --- | --- | +| Nginx 预览 | 只监听 `100.69.2.2:8080`;原有 80 端口站点保持原配置 | +| mirrorouc.service | `mirrorouc` 普通用户,API 与静态文件监听 `127.0.0.1:8081` | +| mirrorouc-config.service | root 受限配置写入器,Unix socket 只允许 `mirrorouc` 组访问 | +| mirrorouc-metrics.timer | 每 10 秒采集 Linux CPU/内存、NFS `/data` I/O 和磁盘容量 | +| mirrorouc-counters.service | 添加独立 nftables 计数表,只计数,不修改已有规则、不丢弃数据包 | +| mirrorouc-speeds.service | 每约 2 秒读取 TCP 接收计数,按仓库汇总并行连接 | +| tunasync | 继续使用已有 Manager `127.0.0.1:12345` 与 Worker `srv-software-mirror` | + +代码发布目录 `/opt/mirrorouc/releases/`;`/opt/mirrorouc/current` 指向当前版本。业务数据库和上传图片保存在本地磁盘 `/var/lib/mirrorouc/`,不要放在 NFS 上。监控历史位于其 `metrics/history.json`,保留最近 24 小时;刚启动时没有完整 24 小时历史。 + +## 首次安装/更新 + +本地 Node.js 22.18+ 与 Go 1.27+: + +```sh +npm ci +npm run check:i18n +npm run check:monitoring +npm run build +cd backend +go test ./... +go vet ./... +GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -o ../mirrorouc . +``` + +把 `mirrorouc`、`dist/` 和 `deploy/` 放进新的 `/opt/mirrorouc/releases//`,确认二进制可执行。在 Debian 服务器上执行: + +```sh +python3 /opt/mirrorouc/releases//deploy/test_config_helper.py +python3 /opt/mirrorouc/releases//deploy/test_job_speeds.py +sh /opt/mirrorouc/releases//deploy/install-metrics.sh +sh /opt/mirrorouc/releases//deploy/install-preview.sh /opt/mirrorouc/releases/ +systemctl status mirrorouc mirrorouc-config mirrorouc-metrics.timer +curl http://127.0.0.1:8081/healthz +nginx -t +``` + +安装脚本针对当前服务器路径。迁移到其他服务器时,应先调整 `nginx-preview.conf`、环境文件、配置写入器的固定路径与 Manager 地址。脚本不会清理旧发布目录或删除镜像文件。 + +首次安装会创建管理员 `admin`,随机密码写入 root 才能读取的 `/etc/mirrorouc/initial-admin-password`。本次部署的本地凭据文件为被 Git 忽略的 `.local/deployment/admin-credentials.json`。后台「账户」可修改当前密码,修改后所有旧会话失效。此密码文件不会跟随密码修改而更新。 + +环境配置文件 `/etc/mirrorouc/mirrorouc.env` 不随发布覆盖,权限 0600。正式启用 HTTPS 后将 `MIRROROUC_SECURE_COOKIES=true` 并重启服务;当前 Tailnet 预览通过 HTTP 访问,故为 false。登录 Cookie 为 HttpOnly、SameSite=Strict,有效期 12 小时;写入接口另需会话 CSRF Token 与同源请求。 + +## 配置发布和权限 + +Web 服务不能直接写 `/etc/tunasync`。配置写入器固定只接受 `/etc/tunasync/worker.conf` 和 `/etc/tunasync/mirrors.d/*.conf`: + +- 全局只可修改 concurrent、interval、retry、timeout。 +- 已有镜像只可修改 upstream、interval、retry、timeout。自定义 command、rsync_options、provider 等字段必须保留。 +- 新增只支持 rsync/two-stage-rsync,文件名固定为 `mirrorouc-.conf`;不接受任意 Shell 命令。 +- 编辑前后均验证 TOML、数值、上游地址和文件内容哈希。产生冲突时拒绝覆盖。 +- 先生成可阅读的完整文件前后预览,再由管理员确认应用;应用失败恢复原文件并尝试重新加载。 +- SQLite 保留变更版本与审计记录;应用过的变更可以恢复至变更前,若文件已被其他修改覆盖则拒绝恢复。 +- 写入器保留待完成事务日志 `/var/lib/mirrorouc-config/pending.json`;若进程在写入中退出,下次启动恢复原文件。若恢复失败则拒绝继续写入,需要运维检查该文件与 Worker 状态。 + +Manager 返回成功表示 reload 命令已提交,并不等于 Worker 已完成重载,需查看 Worker 服务和任务状态。已有仓库禁用后用「开始同步」重新启动;tunasync 不提供单独的 enable 命令。 + +## 监控口径 + +HTTPS/rsync 使用 TCP 443/873 的 IPv4、IPv6 收发字节计数,双向合计,排除回环接口。8080/80 端口的 HTTP 预览流量不计入 HTTPS,也不展示 HTTP 曲线。没有经过这些端口的流量不会被推算进对应协议。没有有效采样时显示等待采集。 + +`/status` 同步明细的「拉取速度」来自 Linux TCP_INFO 的 `bytes_received`,通过 `ss -tinpeH` 读取,关联 `tunasync-worker.service` 的进程与仓库目标路径,并追溯 git/ssh 等辅助进程的父进程。多个 rsync 子进程共享同一个 socket 时按 inode/socket cookie 去重,多个并行连接求和;约每 2 秒采样并刷新,单位 B/s、KB/s、MB/s、GB/s。只计算 TCP 接收,不把上传量、NFS I/O 或磁盘写入速度当作拉取速度。 + +这是采样窗口的接收速率,采样间建立并关闭的短连接可能漏记,不作为精确流量计费数据。首次采样不使用连接生存期累计值冒充瞬时速度。仅同步中的仓库显示数值;有效采样中的零表示当时没有收到数据,未同步/采样失败/数据过期时显示「—」。数据超过 15 秒即失效。相关原始字段见 [Linux TCP_INFO](https://github.com/torvalds/linux/blob/master/include/uapi/linux/tcp.h) 和 [iproute2 ss](https://github.com/iproute2/iproute2/blob/main/misc/ss.c)。 + +NFS I/O 来自 `/proc/self/mountstats` 的 `/data` 挂载,磁盘容量来自 statvfs;CPU、内存来自 `/proc`。采样历史仅覆盖采集器启用后的时间,不补造旧数据。图表最多展示 241 个采样点,均值、当前值、峰值从当前窗口全部原始有效采样计算。 + +镜像文件由 Nginx 直接提供,API 不读取整个仓库;镜像目录响应设置 CSP sandbox 和 nosniff,避免仓库中的 HTML/脚本获得管理页面同源权限。 + +## 备份与回退 + +日常备份包括 SQLite、`uploads/`、`/etc/mirrorouc/` 和 `/etc/tunasync/`。数据库运行时使用 SQLite backup API 或 `sqlite3 ... '.backup ...'`;直接复制数据库文件前先停止 `mirrorouc.service`,同时处理 WAL 文件。备份文件包含账户哈希、会话和可能带内部地址的配置,应限制读取权限。 + +应用版本回退:将 `/opt/mirrorouc/current` 指回旧发布目录,再 `systemctl restart mirrorouc`。本次新增数据库表使用 `CREATE TABLE IF NOT EXISTS`;未来若出现不兼容数据库迁移,需要同时恢复对应数据库备份。 + +只撤下预览时,删除 `/etc/nginx/sites-enabled/mirrorouc-preview`,通过 `nginx -t` 后 reload Nginx,并停止 `mirrorouc.service` 与 `mirrorouc-config.service`。这不会修改原有 80 端口镜像站点。停止新增采集器不会影响 tunasync;若需要删除计数表,只删除 `inet mirrorouc_monitor`,不要 flush 全局 nftables 规则。 + +## 验证范围 + +`backend/*_test.go` 使用临时数据库与假的 Manager,覆盖鉴权、角色、CSRF、草稿隔离、并发冲突、任务提交、配置应用/恢复、上游不可用缓存、图片校验及监控统计。`test_config_helper.py` 独立验证写入白名单、自定义命令保留、注入拒绝与失败恢复,不操作真实同步配置。 + +本次部署核验真实 API、镜像目录、日志、监控与后台草稿保存。没有为验证效果而停止、重启仓库,或应用修改后的生产同步参数。 + +## 统一中文内容升级 + +公告的 title/body 和镜像的 help 已改为单个字符串,所有界面语言共用中文内容。旧版本的语言映射在服务启动时以事务迁移:优先保留 zh-CN,其次 zh;没有中文键的历史记录保留原内容供管理员整理。原始 JSON 保存在 SQLite 的 shared_content_backups 表,迁移会递增记录 revision,避免旧编辑页面覆盖新内容;重复启动不会重复迁移。 + +此次内容格式升级需要在回退旧应用时同时恢复升级前的数据库备份。部署前使用 SQLite backup API 生成一致性快照,备份目录仅 root 可读。迁移不修改镜像介绍、站点文案或同步配置。 + +## 首页相关链接 + +`settings/site` 的 `relatedLinks` 独立控制首页侧栏,与页脚 `links` 分开。旧设置缺少该字段时,接口补充原有的三个入口,不改写数据库。显式保存空数组会隐藏该栏目;仍打开旧版表单的客户端提交时,若未携带该字段,后端保留当前已保存的相关链接。条目支持 `kind=help` 的站内使用帮助与 `kind=url` 的 HTTP(S)/站内绝对路径;不接受脚本、data URL、协议相对地址或反斜杠路径。名称支持多语言,数组顺序即显示顺序。 diff --git a/deploy/collect_job_speeds.py b/deploy/collect_job_speeds.py new file mode 100644 index 0000000..0df05ac --- /dev/null +++ b/deploy/collect_job_speeds.py @@ -0,0 +1,124 @@ +#!/usr/bin/python3 +"""Sample per-socket TCP receive counters and attribute worker processes to mirrors.""" +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import time + +PROC = Path('/proc') +MIRRORS = '/data/mirrors/' +OUTPUT = Path('/var/lib/mirrorouc/metrics/job-speeds.json') +NAME = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,127}$') + +def mirror_name(value): + if not value.startswith(MIRRORS): + return None + name = value[len(MIRRORS):].split('/')[0] + return name if NAME.fullmatch(name) and name not in ('.', '..') else None + +def process_jobs(proc=PROC): + jobs = {} + parents = {} + for entry in proc.iterdir(): + if not entry.name.isdigit(): + continue + try: + if '/tunasync-worker.service' not in (entry / 'cgroup').read_text(): + continue + pid = int(entry.name) + status = (entry / 'stat').read_text().rsplit(') ', 1)[1].split() + parents[pid] = int(status[1]) + arguments = (entry / 'cmdline').read_bytes().decode(errors='replace').split('\0') + candidates = {name for argument in arguments if (name := mirror_name(argument))} + if not candidates: + name = mirror_name(os.readlink(entry / 'cwd')) + if name: + candidates.add(name) + if len(candidates) == 1: + jobs[pid] = candidates.pop() + except (OSError, ValueError, IndexError): + continue + # git/ssh/curl helpers may inherit a job's parent without its destination argument. + for pid in parents: + ancestor = pid + for _ in range(16): + if ancestor in jobs: + jobs[pid] = jobs[ancestor] + break + ancestor = parents.get(ancestor, 0) + if not ancestor: + break + return jobs + +def parse_sockets(output, jobs): + sockets = {} + header = None + for line in output.splitlines(): + if line and not line[0].isspace(): + header = line + continue + if header is None: + continue + identity = re.search(r'\bino:(\d+)\s+sk:([a-fA-F0-9]+)', header) + if not identity: + continue + pids = [int(value) for value in re.findall(r'\bpid=(\d+)', header)] + names = {jobs[pid] for pid in pids if pid in jobs} + received = re.search(r'\bbytes_received:(\d+)', line) + # The same inherited socket can list several PIDs: count its inode/cookie once. + sockets[':'.join(identity.groups())] = {'bytes': int(received.group(1)) if received else 0, 'job': names.pop() if len(names) == 1 else None} + header = None + return sockets + +def rates(current, previous, active, elapsed): + result = {name: {'bytesPerSecond': 0.0, 'connections': 0} for name in active} + if previous is None or elapsed <= 0 or elapsed > 15: + return {} + for identity, socket in current.items(): + name = socket['job'] + if name is None: + continue + old = previous.get(identity) + delta = socket['bytes'] - old['bytes'] if old else socket['bytes'] + if delta < 0: # A reset/reused socket must never produce a negative rate/spike. + continue + value = result.setdefault(name, {'bytesPerSecond': 0.0, 'connections': 0}) + value['bytesPerSecond'] += delta / elapsed + value['connections'] += 1 + return result + +def write(value): + fd, temporary = tempfile.mkstemp(prefix='.speeds-', dir=OUTPUT.parent) + try: + with os.fdopen(fd, 'w') as stream: + json.dump(value, stream, allow_nan=False, separators=(',', ':')) + os.chmod(temporary, 0o640) + os.replace(temporary, OUTPUT) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + +def run(): + previous = None + last = time.monotonic() + while True: + start = time.monotonic() + try: + jobs = process_jobs() + result = subprocess.run(['/usr/bin/ss', '-tinpeH', 'state', 'established'], capture_output=True, text=True, timeout=4, check=True, env={**os.environ, 'LC_ALL': 'C'}) + current = parse_sockets(result.stdout, jobs) + sampled = time.monotonic() + elapsed = sampled - last + values = rates(current, previous, set(jobs.values()), elapsed) + write({'updatedAt': round(time.time() * 1000), 'interval': elapsed, 'available': previous is not None and elapsed <= 15, 'jobs': values}) + previous, last = current, sampled + except (OSError, ValueError, subprocess.SubprocessError): + previous = None + write({'updatedAt': round(time.time() * 1000), 'interval': 2, 'available': False, 'jobs': {}}) + time.sleep(max(.1, 2 - (time.monotonic() - start))) + +if __name__ == '__main__': + run() diff --git a/deploy/collect_metrics.py b/deploy/collect_metrics.py new file mode 100644 index 0000000..e721144 --- /dev/null +++ b/deploy/collect_metrics.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +"""Read Linux/NFS counters. A separate passive nftables table supplies protocol bytes.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time + +ROOT = Path(os.environ.get('MIRROROUC_METRICS_ROOT', '/var/lib/mirrorouc/metrics')) +ROOT.mkdir(parents=True, exist_ok=True) +path = ROOT / 'history.json' +try: + history = json.loads(path.read_text()) +except (OSError, ValueError): + history = {'samples': []} + +now = time.time() +cpu = list(map(int, Path('/proc/stat').read_text().splitlines()[0].split()[1:9])) +mem = {line.split(':')[0]: int(line.split()[1]) * 1024 for line in Path('/proc/meminfo').read_text().splitlines() if len(line.split()) > 1} +nfs = {} +for block in Path('/proc/self/mountstats').read_text().split('device '): + if ' mounted on /data ' not in block: + continue + for line in block.splitlines(): + values = line.strip().split() + if values and values[0] in ('READ:', 'WRITE:'): + nfs[values[0][:-1]] = list(map(int, values[1:])) + +counters = {} +try: + result = subprocess.run(['/usr/sbin/nft', '-j', 'list', 'counters', 'table', 'inet', 'mirrorouc_monitor'], capture_output=True, timeout=4, check=True) + for item in json.loads(result.stdout).get('nftables', []): + counter = item.get('counter', {}) + if counter.get('name') in ('https4', 'https6', 'rsync4', 'rsync6'): + counters[counter['name']] = counter['bytes'] +except (OSError, ValueError, subprocess.SubprocessError): + pass + +previous = history.get('previous') +sample = {'time': round(now * 1000)} +if previous: + elapsed = max(.1, now - previous['time']) + delta = [max(0, a - b) for a, b in zip(cpu, previous['cpu'])] + total = sum(delta) + if total: + sample.update(user=(delta[0] + delta[1]) / total * 100, system=(delta[2] + delta[5] + delta[6] + delta[7]) / total * 100, wait=delta[4] / total * 100, idle=delta[3] / total * 100) + for key, value in counters.items(): + old = previous.get('counters', {}).get(key) + if old is not None and value >= old: + sample[key] = (value - old) * 8 / elapsed / 1e9 + for operation, prefix in [('READ', 'read'), ('WRITE', 'write')]: + a, b = nfs.get(operation), previous.get('nfs', {}).get(operation) + if a and b and a[0] >= b[0]: + sample[prefix + 'Ops'] = (a[0] - b[0]) / elapsed / 1000 + byte_index = 4 if operation == 'READ' else 3 + sample[prefix + 'Bytes'] = max(0, a[byte_index] - b[byte_index]) / elapsed / 1e9 + cached = max(0, mem.get('Cached', 0) + mem.get('SReclaimable', 0) - mem.get('Shmem', 0)) + free, buffers, total_mem = mem['MemFree'], mem.get('Buffers', 0), mem['MemTotal'] + used = max(0, total_mem - free - cached - buffers) + sample.update(used=used / 2**30, cached=cached / 2**30, buffers=buffers / 2**30, free=free / 2**30) + if 'user' in sample: + history['samples'].append(sample) + +disks = [] +for mount in ['/data', '/']: + try: + disk = os.statvfs(mount) + disks.append({'name': mount, 'total': disk.f_blocks * disk.f_frsize / 1e12, 'used': (disk.f_blocks - disk.f_bfree) * disk.f_frsize / 1e12}) + except OSError: + pass +history.update(previous={'time': now, 'cpu': cpu, 'nfs': nfs, 'counters': counters}, updatedAt=round(now * 1000), memoryTotal=mem['MemTotal'] / 2**30, disks=disks) +history['samples'] = [point for point in history['samples'] if point['time'] >= (now - 86400) * 1000][-8640:] +fd, temporary = tempfile.mkstemp(dir=ROOT, prefix='.metrics-') +try: + with os.fdopen(fd, 'w') as output: + json.dump(history, output, allow_nan=False, separators=(',', ':')) + output.flush() + os.fsync(output.fileno()) + os.chmod(temporary, 0o640) + os.replace(temporary, path) +finally: + if os.path.exists(temporary): + os.unlink(temporary) diff --git a/deploy/config-helper.py b/deploy/config-helper.py new file mode 100644 index 0000000..66425f9 --- /dev/null +++ b/deploy/config-helper.py @@ -0,0 +1,178 @@ +#!/usr/bin/python3 +"""Root-only configuration writer, reachable only through a group-restricted Unix socket. + +Independently verifies allowed TOML changes; never executes submitted commands. +""" +import copy +import glob +import grp +import hashlib +import json +import os +import re +import socketserver +import tempfile +import tomllib +import urllib.parse +import urllib.request + +ROOT = '/etc/tunasync' +MAIN = ROOT + '/worker.conf' +INCLUDES = ROOT + '/mirrors.d' +SOCKET = '/run/mirrorouc-config/control.sock' +JOURNAL = '/var/lib/mirrorouc-config/pending.json' +IDENTIFIER = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_.+-]{0,127}$') + +def number(value, lower, upper): + return type(value) is int and lower <= value <= upper + +def upstream(value): + if not isinstance(value, str) or len(value) >= 2048 or re.search(r'[\s`;$|<>\\\"\']', value): + return False + parsed = urllib.parse.urlsplit(value) + return parsed.scheme in ('rsync', 'https', 'http') and bool(parsed.hostname) and parsed.username is None + +def sync_numbers(table): + return all(key not in table or number(table[key], low, high) for key, low, high in [('interval', 1, 525600), ('retry', 0, 20), ('timeout', 30, 1209600)]) + +def stripped_mirrors(doc): + result = copy.deepcopy(doc) + mirrors = result.get('mirrors', []) + assert isinstance(mirrors, list) + for mirror in mirrors: + for field in ('upstream', 'interval', 'retry', 'timeout'): + mirror.pop(field, None) + return result + +def validate_new(doc, path): + assert set(doc) == {'mirrors'} and len(doc['mirrors']) == 1 + mirror = doc['mirrors'][0] + assert set(mirror) <= {'name', 'provider', 'upstream', 'interval', 'retry', 'timeout', 'stage1_profile'} + assert IDENTIFIER.fullmatch(mirror['name']) and mirror['name'] not in ('.', '..') + assert path == INCLUDES + '/mirrorouc-' + mirror['name'] + '.conf' + assert mirror['provider'] in ('rsync', 'two-stage-rsync') + assert upstream(mirror['upstream']) and sync_numbers(mirror) + assert 'stage1_profile' not in mirror or mirror['stage1_profile'] == 'debian' + +def validate(path, before, after): + assert path == MAIN or (os.path.dirname(path) == INCLUDES and os.path.basename(path).endswith('.conf')) + assert not os.path.islink(path) + old = tomllib.loads(before) if before is not None else None + new = tomllib.loads(after) if after is not None else None + assert old is not None or new is not None + if path == MAIN: + assert old is not None and new is not None + assert number(new['global'].get('concurrent'), 1, 64) and sync_numbers(new['global']) + for doc in (old, new): + for field in ('concurrent', 'interval', 'retry', 'timeout'): + doc['global'].pop(field, None) + assert old == new + elif old is None or new is None: + validate_new(new if new is not None else old, path) + else: + assert stripped_mirrors(old) == stripped_mirrors(new) + for previous, current in zip(old.get('mirrors', []), new.get('mirrors', [])): + if previous != current: + assert upstream(current['upstream']) and sync_numbers(current) + +def read(path): + try: + with open(path, encoding='utf-8', newline='') as stream: + return stream.read() + except FileNotFoundError: + return None + +def atomic(path, value): + if value is None: + os.unlink(path) + return + mode = os.stat(path).st_mode & 0o777 if os.path.exists(path) else 0o644 + fd, temporary = tempfile.mkstemp(prefix='.mirrorouc-', dir=os.path.dirname(path)) + try: + with os.fdopen(fd, 'w', encoding='utf-8', newline='') as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary, mode) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + +def reload_worker(): + worker = tomllib.loads(read(MAIN))['global']['name'] + body = json.dumps({'worker_id': worker, 'mirror_id': '', 'cmd': 'reload', 'args': [], 'options': {}}).encode() + request = urllib.request.Request('http://127.0.0.1:12345/cmd', data=body, headers={'Content-Type': 'application/json'}) + with urllib.request.urlopen(request, timeout=10) as response: + assert response.status == 200 + +def signature(): + pattern = tomllib.loads(read(MAIN)).get('include', {}).get('include_mirrors', '') + assert pattern == INCLUDES + '/*.conf' + names = sorted(set([MAIN] + glob.glob(pattern))) + digest = hashlib.sha256() + for path in names: + assert not os.path.islink(path) + raw = read(path).encode() + digest.update(str(len(path.encode())).encode() + b':' + path.encode() + str(len(raw)).encode() + b':' + raw) + return digest.hexdigest() + +def restore(changes): + for path, change in changes.items(): + if change['before'] is None and not os.path.exists(path): + continue + atomic(path, change['before']) + reload_worker() + +def apply(request): + changes = request['files'] + assert isinstance(changes, dict) and 0 < len(changes) <= 20 + for path, change in changes.items(): + validate(path, change['before'], change['after']) + if read(path) != change['before']: + return 'CONFLICT' + if signature() != request['baseHash']: + return 'CONFLICT' + # Write-ahead journal survives a helper/process restart during a multi-file update. + atomic(JOURNAL, json.dumps(changes)) + try: + for path, change in changes.items(): + atomic(path, change['after']) + reload_worker() + except Exception: + try: + restore(changes) + except Exception: + return 'CONFIG_ROLLBACK_FAILED' + os.unlink(JOURNAL) + return 'CONFIG_APPLY_FAILED' + os.unlink(JOURNAL) + return '' + +class Handler(socketserver.StreamRequestHandler): + def handle(self): + self.connection.settimeout(5) + try: + raw = self.rfile.readline(2 * 1024 * 1024 + 1) + assert len(raw) <= 2 * 1024 * 1024 and raw.endswith(b'\n') + if os.path.exists(JOURNAL): + code = 'CONFIG_ROLLBACK_FAILED' + else: + code = apply(json.loads(raw)) + except (AssertionError, KeyError, TypeError, ValueError): + code = 'INVALID_INPUT' + except Exception: + code = 'CONFIG_APPLY_FAILED' + self.wfile.write(json.dumps({'code': code}).encode() + b'\n') + +if __name__ == '__main__': + if os.path.exists(JOURNAL): + with open(JOURNAL) as stream: + restore(json.load(stream)) + os.unlink(JOURNAL) + if os.path.exists(SOCKET): + os.unlink(SOCKET) + with socketserver.UnixStreamServer(SOCKET, Handler) as server: + os.chown(SOCKET, 0, grp.getgrnam('mirrorouc').gr_gid) + os.chmod(SOCKET, 0o660) + server.serve_forever() diff --git a/deploy/ensure-counters.sh b/deploy/ensure-counters.sh new file mode 100644 index 0000000..ad91b98 --- /dev/null +++ b/deploy/ensure-counters.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +if ! /usr/sbin/nft list table inet mirrorouc_monitor >/dev/null 2>&1; then + /usr/sbin/nft -f /etc/mirrorouc/protocol-counters.nft +fi diff --git a/deploy/install-metrics.sh b/deploy/install-metrics.sh new file mode 100644 index 0000000..f4d717c --- /dev/null +++ b/deploy/install-metrics.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")" +if ! id mirrorouc >/dev/null 2>&1; then + useradd --system --home-dir /var/lib/mirrorouc --shell /usr/sbin/nologin mirrorouc +fi +install -d -o mirrorouc -g mirrorouc -m 0700 /var/lib/mirrorouc +install -d -o root -g mirrorouc -m 0750 /var/lib/mirrorouc/metrics +install -d -o root -g root -m 0755 /usr/local/lib/mirrorouc +install -d -o root -g root -m 0755 /etc/mirrorouc +install -m 0644 collect_metrics.py /usr/local/lib/mirrorouc/collect_metrics.py +install -m 0644 collect_job_speeds.py /usr/local/lib/mirrorouc/collect_job_speeds.py +install -m 0755 ensure-counters.sh /usr/local/lib/mirrorouc/ensure-counters.sh +install -m 0644 protocol-counters.nft /etc/mirrorouc/protocol-counters.nft +install -m 0644 mirrorouc-counters.service mirrorouc-metrics.service mirrorouc-metrics.timer mirrorouc-speeds.service /etc/systemd/system/ +if ! /usr/sbin/nft list table inet mirrorouc_monitor >/dev/null 2>&1; then + /usr/sbin/nft -c -f /etc/mirrorouc/protocol-counters.nft +fi +systemctl daemon-reload +systemctl enable --now mirrorouc-counters.service mirrorouc-metrics.timer +systemctl start mirrorouc-metrics.service +systemctl enable --now mirrorouc-speeds.service +systemctl restart mirrorouc-speeds.service diff --git a/deploy/install-preview.sh b/deploy/install-preview.sh new file mode 100644 index 0000000..bb68b4f --- /dev/null +++ b/deploy/install-preview.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Run as root: sh deploy/install-preview.sh /opt/mirrorouc/releases/ +set -eu +release=$(readlink -f "$1") +case "$release" in /opt/mirrorouc/releases/*) ;; *) echo 'Invalid release path' >&2; exit 1;; esac +test -x "$release/mirrorouc" +test -f "$release/dist/index.html" +deploy="$release/deploy" +id mirrorouc >/dev/null 2>&1 || useradd --system --home-dir /var/lib/mirrorouc --shell /usr/sbin/nologin mirrorouc +install -d -m 0700 -o mirrorouc -g mirrorouc /var/lib/mirrorouc +install -d -m 0755 /etc/mirrorouc /usr/local/lib/mirrorouc +install -d -m 0700 /var/lib/mirrorouc-config +install -m 0755 "$deploy/config-helper.py" /usr/local/lib/mirrorouc/config-helper.py +install -m 0644 "$deploy/mirrorouc.service" "$deploy/mirrorouc-config.service" /etc/systemd/system/ +if [ ! -f /etc/mirrorouc/mirrorouc.env ]; then + umask 077 + cat > /etc/mirrorouc/mirrorouc.env <<'EOF' +MIRROROUC_LISTEN=127.0.0.1:8081 +MIRROROUC_DATA=/var/lib/mirrorouc +MIRROROUC_STATIC=/opt/mirrorouc/current/dist +MIRROROUC_MANAGER=http://127.0.0.1:12345 +MIRROROUC_WORKER=srv-software-mirror +MIRROROUC_WORKER_CONFIG=/etc/tunasync/worker.conf +MIRROROUC_LOGS=/var/log/tunasync +MIRROROUC_METRICS=/var/lib/mirrorouc/metrics/history.json +MIRROROUC_CONFIG_SOCKET=/run/mirrorouc-config/control.sock +MIRROROUC_SECURE_COOKIES=false +EOF +fi +# Leave current tunasync configurations, ownership and permissions intact. +if [ ! -s /var/lib/mirrorouc/mirrorouc.db ]; then + umask 077 + runuser -u mirrorouc -- "$release/mirrorouc" -data /var/lib/mirrorouc -init-admin admin > /etc/mirrorouc/initial-admin-password +fi +ln -sfn "$release" /opt/mirrorouc/current.new +mv -Tf /opt/mirrorouc/current.new /opt/mirrorouc/current +install -m 0644 "$deploy/nginx-preview.conf" /etc/nginx/sites-available/mirrorouc-preview +ln -sfn /etc/nginx/sites-available/mirrorouc-preview /etc/nginx/sites-enabled/mirrorouc-preview +nginx -t +systemctl daemon-reload +systemctl enable --now mirrorouc-config.service +systemctl restart mirrorouc-config.service +systemctl enable mirrorouc.service +systemctl restart mirrorouc.service +systemctl reload nginx +echo 'MirrorOUC preview installed at http://100.69.2.2:8080' diff --git a/deploy/mirrorouc-config.service b/deploy/mirrorouc-config.service new file mode 100644 index 0000000..5359023 --- /dev/null +++ b/deploy/mirrorouc-config.service @@ -0,0 +1,25 @@ +[Unit] +Description=MirrorOUC restricted tunasync configuration writer +After=tunasync-manager.service + +[Service] +ExecStart=/usr/bin/python3 /usr/local/lib/mirrorouc/config-helper.py +Restart=on-failure +RestartSec=5 +RuntimeDirectory=mirrorouc-config +RuntimeDirectoryMode=0755 +StateDirectory=mirrorouc-config +StateDirectoryMode=0700 +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/etc/tunasync /var/lib/mirrorouc-config /run/mirrorouc-config +ProtectKernelTunables=true +ProtectControlGroups=true +RestrictSUIDSGID=true +RestrictAddressFamilies=AF_UNIX AF_INET + +[Install] +WantedBy=multi-user.target diff --git a/deploy/mirrorouc-counters.service b/deploy/mirrorouc-counters.service new file mode 100644 index 0000000..804c84b --- /dev/null +++ b/deploy/mirrorouc-counters.service @@ -0,0 +1,9 @@ +[Unit] +Description=MirrorOUC passive protocol byte counters +After=nftables.service +[Service] +Type=oneshot +ExecStart=/usr/local/lib/mirrorouc/ensure-counters.sh +RemainAfterExit=yes +[Install] +WantedBy=multi-user.target diff --git a/deploy/mirrorouc-metrics.service b/deploy/mirrorouc-metrics.service new file mode 100644 index 0000000..3386d26 --- /dev/null +++ b/deploy/mirrorouc-metrics.service @@ -0,0 +1,13 @@ +[Unit] +Description=MirrorOUC read-only host metrics collection +[Service] +Type=oneshot +User=root +Group=mirrorouc +ExecStart=/usr/bin/python3 /usr/local/lib/mirrorouc/collect_metrics.py +UMask=0027 +NoNewPrivileges=true +ProtectSystem=strict +ReadWritePaths=/var/lib/mirrorouc/metrics +ProtectHome=true +PrivateTmp=true diff --git a/deploy/mirrorouc-metrics.timer b/deploy/mirrorouc-metrics.timer new file mode 100644 index 0000000..c334859 --- /dev/null +++ b/deploy/mirrorouc-metrics.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Collect MirrorOUC metrics every 10 seconds +[Timer] +OnBootSec=10 +OnUnitActiveSec=10 +AccuracySec=1 +[Install] +WantedBy=timers.target diff --git a/deploy/mirrorouc-speeds.service b/deploy/mirrorouc-speeds.service new file mode 100644 index 0000000..2d2f0f9 --- /dev/null +++ b/deploy/mirrorouc-speeds.service @@ -0,0 +1,22 @@ +[Unit] +Description=MirrorOUC per-mirror upstream TCP receive rates +After=tunasync-worker.service + +[Service] +User=root +Group=mirrorouc +ExecStart=/usr/bin/python3 /usr/local/lib/mirrorouc/collect_job_speeds.py +Restart=on-failure +RestartSec=3 +UMask=0027 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/var/lib/mirrorouc/metrics +ProtectKernelTunables=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_UNIX AF_NETLINK AF_INET AF_INET6 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/mirrorouc.service b/deploy/mirrorouc.service new file mode 100644 index 0000000..bd2c213 --- /dev/null +++ b/deploy/mirrorouc.service @@ -0,0 +1,26 @@ +[Unit] +Description=MirrorOUC web and administration API +After=network.target tunasync-manager.service mirrorouc-config.service +Wants=mirrorouc-config.service + +[Service] +User=mirrorouc +Group=mirrorouc +WorkingDirectory=/opt/mirrorouc/current +EnvironmentFile=/etc/mirrorouc/mirrorouc.env +ExecStart=/opt/mirrorouc/current/mirrorouc +Restart=on-failure +RestartSec=5 +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/mirrorouc +ProtectKernelTunables=true +ProtectControlGroups=true +RestrictSUIDSGID=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/nginx-preview.conf b/deploy/nginx-preview.conf new file mode 100644 index 0000000..465c1c1 --- /dev/null +++ b/deploy/nginx-preview.conf @@ -0,0 +1,22 @@ +server { + listen 100.69.2.2:8080; + server_name 100.69.2.2; + client_max_body_size 5m; + + location /mirrors/ { + alias /data/mirrors/; + autoindex on; + autoindex_exact_size off; + autoindex_localtime on; + add_header X-Content-Type-Options nosniff always; + add_header Content-Security-Policy "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src 'self' data:" always; + } + + location / { + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 35s; + } +} diff --git a/deploy/protocol-counters.nft b/deploy/protocol-counters.nft new file mode 100644 index 0000000..1c0d31b --- /dev/null +++ b/deploy/protocol-counters.nft @@ -0,0 +1,25 @@ +# Passive accounting only; no existing rules or tables are flushed. +table inet mirrorouc_monitor { + counter https4 { } + counter https6 { } + counter rsync4 { } + counter rsync6 { } + chain account { + meta nfproto ipv4 tcp sport 443 counter name https4 + meta nfproto ipv4 tcp dport 443 tcp sport != 443 counter name https4 + meta nfproto ipv6 tcp sport 443 counter name https6 + meta nfproto ipv6 tcp dport 443 tcp sport != 443 counter name https6 + meta nfproto ipv4 tcp sport 873 counter name rsync4 + meta nfproto ipv4 tcp dport 873 tcp sport != 873 counter name rsync4 + meta nfproto ipv6 tcp sport 873 counter name rsync6 + meta nfproto ipv6 tcp dport 873 tcp sport != 873 counter name rsync6 + } + chain input { + type filter hook input priority 300; policy accept; + iifname != "lo" jump account + } + chain output { + type filter hook output priority 300; policy accept; + oifname != "lo" jump account + } +} diff --git a/deploy/test_config_helper.py b/deploy/test_config_helper.py new file mode 100644 index 0000000..bd5bf1d --- /dev/null +++ b/deploy/test_config_helper.py @@ -0,0 +1,47 @@ +import copy +import importlib.util +import json +import os +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('helper', os.path.join(os.path.dirname(__file__), 'config-helper.py')) +helper = importlib.util.module_from_spec(spec) +spec.loader.exec_module(helper) + +class ValidationTests(unittest.TestCase): + def test_preserves_custom_commands(self): + before = '[[mirrors]]\nname="debian"\nprovider="rsync"\nupstream="rsync://old/debian"\ncommand="/usr/local/bin/parallel-rsync"\n' + after = before.replace('old/debian', 'new/debian') + 'interval=60\nretry=2\ntimeout=120\n' + helper.validate(helper.INCLUDES + '/debian.conf', before, after) + with self.assertRaises(AssertionError): + helper.validate(helper.INCLUDES + '/debian.conf', before, after.replace('parallel-rsync', 'other-command')) + + def test_blocks_other_global_fields_and_paths(self): + before = '[global]\nname="worker"\nconcurrent=12\ninterval=60\nretry=2\ntimeout=120\n' + helper.validate(helper.MAIN, before, before.replace('concurrent=12', 'concurrent=8')) + for path, after in [('/etc/shadow', before), (helper.MAIN, before.replace('worker', 'other')), (helper.MAIN, before.replace('concurrent=12', 'concurrent=0'))]: + with self.assertRaises(AssertionError): + helper.validate(path, before, after) + + def test_new_mirrors_and_injection(self): + value = '[[mirrors]]\nname="test"\nprovider="rsync"\nupstream="rsync://mirror/test"\ninterval=60\nretry=0\ntimeout=120\n' + helper.validate_new(helper.tomllib.loads(value), helper.INCLUDES + '/mirrorouc-test.conf') + for changed in [value + 'command="evil"\n', value.replace('rsync://mirror/test', 'rsync://mirror/`cmd`'), value.replace('retry=0', 'retry=-1')]: + with self.assertRaises(AssertionError): + helper.validate_new(helper.tomllib.loads(changed), helper.INCLUDES + '/mirrorouc-test.conf') + + def test_reload_failure_restores_files(self): + with tempfile.TemporaryDirectory() as root: + path = root + '/worker.conf' + before = '[global]\nname="worker"\nconcurrent=12\ninterval=60\nretry=2\ntimeout=120\n' + helper.atomic(path, before) + changes = {path: {'before': before, 'after': before.replace('concurrent=12', 'concurrent=8')}} + with patch.object(helper, 'MAIN', path), patch.object(helper, 'JOURNAL', root + '/pending.json'), patch.object(helper, 'signature', return_value='base'), patch.object(helper, 'reload_worker', side_effect=[OSError('unavailable'), None]): + self.assertEqual(helper.apply({'files': changes, 'baseHash': 'base'}), 'CONFIG_APPLY_FAILED') + self.assertEqual(helper.read(path), before) + self.assertFalse(os.path.exists(root + '/pending.json')) + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/test_job_speeds.py b/deploy/test_job_speeds.py new file mode 100644 index 0000000..c176631 --- /dev/null +++ b/deploy/test_job_speeds.py @@ -0,0 +1,40 @@ +import importlib.util +import os +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('speeds', os.path.join(os.path.dirname(__file__), 'collect_job_speeds.py')) +speeds = importlib.util.module_from_spec(spec) +spec.loader.exec_module(speeds) + +class SpeedTests(unittest.TestCase): + def test_sockets_deduplicate_shared_pids_and_separate_repos(self): + data = '0 0 10.0.0.1:123 10.0.0.2:873 users:(("rsync",pid=1,fd=3),("rsync",pid=2,fd=3)) uid:9 ino:20 sk:abc <->\n\t cubic bytes_sent:4 bytes_received:1000\n0 0 10.0.0.1:124 10.0.0.2:873 users:(("rsync",pid=3,fd=3)) uid:9 ino:21 sk:abd <->\n\t cubic bytes_received:2000\n' + current = speeds.parse_sockets(data, {1: 'debian', 2: 'debian', 3: 'ubuntu'}) + result = speeds.rates(current, {'20:abc': {'bytes': 400}, '21:abd': {'bytes': 1000}}, {'debian', 'ubuntu'}, 2) + self.assertEqual(result['debian'], {'bytesPerSecond': 300, 'connections': 1}) + self.assertEqual(result['ubuntu']['bytesPerSecond'], 500) + + def test_parallel_and_new_connections_add_without_startup_spike(self): + current = {'1:a': {'bytes': 1000, 'job': 'debian'}, '2:b': {'bytes': 500, 'job': 'debian'}, '3:c': {'bytes': 9000, 'job': None}} + self.assertEqual(speeds.rates(current, None, {'debian'}, 2), {}) + result = speeds.rates(current, {'1:a': {'bytes': 800}}, {'debian', 'ubuntu'}, 2) + self.assertEqual(result['debian']['bytesPerSecond'], 350) + self.assertEqual(result['ubuntu']['bytesPerSecond'], 0) + self.assertEqual(speeds.rates(current, {}, {'debian'}, 20), {}) + + def test_process_ancestry_and_path_boundaries(self): + with tempfile.TemporaryDirectory() as root: + for pid, parent, command in [(1, 0, ['/usr/bin/python', '/worker']), (2, 1, ['rsync', '/data/mirrors/debian/']), (3, 2, ['ssh', 'host']), (4, 0, ['rsync', '/data/mirrors/ubuntu/'])]: + path = Path(root) / str(pid); path.mkdir() + (path/'stat').write_text(f'{pid} (process) S {parent} 0 0') + (path/'cgroup').write_text('0::/system.slice/tunasync-worker.service' if pid != 4 else '0::/system.slice/other.service') + (path/'cmdline').write_bytes('\0'.join(command).encode()) + jobs = speeds.process_jobs(Path(root)) + self.assertEqual(jobs, {2: 'debian', 3: 'debian'}) + self.assertIsNone(speeds.mirror_name('/data/mirrors-other/secret')) + self.assertIsNone(speeds.mirror_name('/data/mirrors/../secret')) + +if __name__ == '__main__': + unittest.main() diff --git a/package-lock.json b/package-lock.json index 71f0157..b660b72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,9 @@ "name": "mirrorouc-webui", "version": "0.1.0", "dependencies": { + "dompurify": "^3.4.17", "lucide-vue-next": "^0.468.0", + "marked": "^18.1.2", "vue": "^3.5.22", "vue-i18n": "^11.1.12" }, @@ -1014,6 +1016,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT", + "optional": true + }, "node_modules/@vitejs/plugin-vue": { "version": "6.0.10", "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.10.tgz", @@ -1223,6 +1232,15 @@ "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, + "node_modules/dompurify": { + "version": "3.4.17", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.17.tgz", + "integrity": "sha512-B/pSlYVHQ58kqISD/yDjuJ1yfzI34m1gF4lAod2bwSRhiN4K5Cl3H8Af7hqB2PmTL911tdEudnnXn5FnhRoz0w==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, "node_modules/entities": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", @@ -1335,6 +1353,18 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/marked": { + "version": "18.1.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-18.1.2.tgz", + "integrity": "sha512-DfOnvLAQ6WYeWxWO1u4RyFOeqyHup6dks5PPqnoFiooU7ipRvidO/2j8plhAHIiZg574E9ZlzSjUiPMLAzFkgg==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, "node_modules/muggle-string": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/muggle-string/-/muggle-string-0.4.1.tgz", diff --git a/package.json b/package.json index 58d828c..04eda08 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,9 @@ "private": true, "version": "0.1.0", "type": "module", - "engines": { "node": ">=22.18.0" }, + "engines": { + "node": ">=22.18.0" + }, "scripts": { "dev": "vite --host 127.0.0.1", "build": "vue-tsc -b && vite build", @@ -12,7 +14,9 @@ "check:monitoring": "node scripts/check-monitoring.ts" }, "dependencies": { + "dompurify": "^3.4.17", "lucide-vue-next": "^0.468.0", + "marked": "^18.1.2", "vue": "^3.5.22", "vue-i18n": "^11.1.12" }, diff --git a/plan.md b/plan.md index f7fab18..6fae879 100644 --- a/plan.md +++ b/plan.md @@ -1,5 +1,7 @@ ## 已确定的方向 +> 实现进度:前台、Go HTTP + SQLite 管理服务及 Vue 管理页面已接入。当前预览部署、使用方式与实际权限边界以 [README.md](README.md) 和 [deploy/README.md](deploy/README.md) 为准;下文保留原始方案。 + - 使用 Vue 3、TypeScript 和 Vite 开发前端。 - 使用 vue-i18n,所有界面文案通过语言键引用语言文件。 - 第一版支持简体中文(`zh-CN`)和英文(`en-US`)。 @@ -69,7 +71,7 @@ archlinux 58 分钟前 / 即将开始 1.3 TB / 等待同步│ 相关 | 模块 | 计划内容 | | --- | --- | -| 公告与帮助 | Markdown 编辑、预览、草稿、发布、置顶及多语言内容 | +| 公告与帮助 | Markdown 编辑、预览、草稿、发布、置顶;公告与镜像帮助统一中文内容 | | 镜像信息 | 名称、分类、描述、帮助关联、公开状态与下载入口 | | 同步配置 | 上游、同步方式、同步间隔、重试、超时及并发设置 | | 任务操作 | 手动触发、停止、执行结果、日志和同步历史 | @@ -162,7 +164,7 @@ export default { ### 动态内容 -公告、帮助文档、镜像介绍和可编辑站点文案按语言保存于内容数据中,不写入前端语言包。缺少译文时展示原始语言内容,并标明内容语言。 +公告标题、正文和镜像帮助文档只维护一份中文,所有界面语言展示相同内容,正文节点标记 `lang="zh-CN"`。镜像介绍和可编辑站点文案仍按语言保存于内容数据中,不写入前端语言包。 ## 建议技术架构 @@ -242,7 +244,7 @@ tunasync 的原生 `interval` 表示同步完成后等待指定分钟数再执 - 核对服务器环境、tunasync 版本和现有配置。 - 建立管理服务与状态适配层,接入真实镜像列表。 - 完成公告、帮助、常用下载和站点设置。 -- 完成后台登录、内容编辑与多语言内容存储。 +- 完成后台登录、内容编辑;公告和镜像帮助统一中文,其他站点文案按需支持多语言。 - 配置镜像文件服务、网站路由与部署流程。 ### 第三阶段:运维管理 diff --git a/scripts/check-locales.ts b/scripts/check-locales.ts index 0ea9401..150b51a 100644 --- a/scripts/check-locales.ts +++ b/scripts/check-locales.ts @@ -9,6 +9,7 @@ import monitorEn from '../src/locales/en-US/monitor.ts' import { datetimeFormats as datetimeZh, numberFormats as numbersZh } from '../src/locales/zh-CN/formats.ts' import { datetimeFormats as datetimeEn, numberFormats as numbersEn } from '../src/locales/en-US/formats.ts' import { createI18n } from 'vue-i18n' +import { adminZh, adminEn } from '../src/locales/admin-messages.ts' function flatten(value: Record, prefix = ''): Record { return Object.fromEntries(Object.entries(value).flatMap(([key, entry]) => { @@ -17,8 +18,8 @@ function flatten(value: Record, prefix = ''): Record articleRecords.value.map(row => ({ id: row.id, ...row.data }))) +const distributions = computed(() => site.value?.downloads || []) +const description = (mirror: Mirror) => content(mirror.description, locale.value) const language = computed(() => locale.value as Locale) -const localeMessage = (key: string, selectedLocale: Locale = language.value) => t(key, {}, { locale: selectedLocale }) type Theme = 'light' | 'dark' | 'system' const storedTheme = readPreference('mirrorouc.theme') const theme = ref(storedTheme === 'light' || storedTheme === 'dark' ? storedTheme : 'system') const systemDark = window.matchMedia('(prefers-color-scheme: dark)') const prefersDark = ref(systemDark.matches) const dark = computed(() => theme.value === 'dark' || (theme.value === 'system' && prefersDark.value)) -const brandLogo = computed(() => dark.value ? '/assets/mirrorOUC.png' : '/assets/mirrorOUC_dark.png') +const brandLogo = computed(() => dark.value ? site.value?.logoDark || '/assets/mirrorOUC.png' : site.value?.logoLight || '/assets/mirrorOUC_dark.png') +watch(() => site.value?.favicon, value => { if (value) document.querySelector('link[rel="icon"]')?.setAttribute('href', value) }) const themeOpen = ref(false) const mobileOpen = ref(false) const themeRoot = ref() @@ -30,7 +39,7 @@ const themes: Theme[] = ['light', 'dark', 'system'] const themeIcons = { light: Sun, dark: Moon, system: Monitor } const isStatusPage = window.location.pathname.replace(/\/$/, '') === '/status' const navigation = [ - { key: 'mirrors', href: '/', active: !isStatusPage }, + { key: 'mirrors', href: '/', active: !isStatusPage && !isAdminPage }, { key: 'news', href: 'https://www.itouc.cn/announcements/', active: false }, { key: 'status', href: '/status', active: isStatusPage }, { key: 'feedback', href: 'https://git.itouc.cn/ITStudio_OUC/OUCMirror-WebUI/issues', active: false }, @@ -46,16 +55,16 @@ watch(locale, (value) => { document.title = t('common.pageTitle') savePreference('mirrorouc.locale', value) }, { immediate: true }) +watch([locale, site], () => { document.title = site.value ? `MirrorOUC · ${content(site.value.name, locale.value)}` : t('common.pageTitle') }) function selectTheme(value: Theme) { theme.value = value; themeOpen.value = false } const search = ref('') const searchInput = ref() const statusPage = ref>() -const mirrors = ref(demoMirrors.map(mirror => ({ ...mirror }))) const filteredMirrors = computed(() => { const query = search.value.trim().toLowerCase() if (!query) return mirrors.value - return mirrors.value.filter(mirror => [mirror.name, ...supportedLocales.map(value => localeMessage(mirror.descriptionKey, value))].join(' ').toLowerCase().includes(query)) + return mirrors.value.filter(mirror => [mirror.name, ...Object.values(mirror.description || {})].join(' ').toLowerCase().includes(query)) }) const now = ref(Date.now()) const { mirrorSize, relativeTime, tableNextTime, fullTime } = useMirrorFormats(now) @@ -76,27 +85,16 @@ function nextLabel(mirror: Mirror) { ? t(`mirrors.status.${mirror.status}`) : relativeTime(mirror.nextSync, true) } function resetSearch() { search.value = ''; searchInput.value?.focus() } -let demoStep = 0 -function advanceDemo() { - demoStep++ - const mirror = mirrors.value.find(item => item.name === (demoStep % 2 ? 'anaconda' : 'debian'))! - mirror.status = mirror.status === 'syncing' ? 'success' : 'syncing' - mirror.lastAttempt = Date.now() - if (mirror.status === 'success') { mirror.lastSuccess = mirror.lastAttempt; mirror.nextSync = mirror.lastAttempt + 60 * 60_000; mirror.syncStarted = null } - else { mirror.nextSync = null; mirror.syncStarted = mirror.lastAttempt } -} type Modal = 'help' | 'news' | 'about' | 'downloads' | 'mirror' const dialog = ref() const modal = ref(null) const selectedMirror = ref(null) const selectedNews = ref(null) -const copied = ref(false) -const copyError = ref(false) -const activeNews = computed(() => announcements.find(item => item.id === selectedNews.value)) +const activeNews = computed(() => announcements.value.find(item => item.id === selectedNews.value)) const modalTitle = computed(() => { if (modal.value === 'mirror') return selectedMirror.value?.name || '' - if (modal.value === 'news' && activeNews.value) return t(activeNews.value.titleKey) + if (modal.value === 'news' && activeNews.value) return activeNews.value.title const keys = { help: 'mirrors.helpTitle', news: 'announcements.title', about: 'common.aboutTitle', downloads: 'announcements.downloadsTitle' } return modal.value && modal.value in keys ? t(keys[modal.value as keyof typeof keys]) : '' }) @@ -104,8 +102,6 @@ async function openModal(value: Modal, mirror?: Mirror, news?: string) { modal.value = value selectedMirror.value = mirror || null selectedNews.value = news || null - copied.value = false - copyError.value = false mobileOpen.value = false themeOpen.value = false await nextTick() @@ -114,20 +110,6 @@ async function openModal(value: Modal, mirror?: Mirror, news?: string) { function closeModal() { dialog.value?.close() } function onDialogClose() { modal.value = null; selectedMirror.value = null } watch(modal, value => { document.body.classList.toggle('dialog-open', value !== null) }) -const configExample = computed(() => { - switch (selectedMirror.value?.name) { - case 'ubuntu': return 'deb https://example.invalid/ubuntu/ noble main universe' - case 'debian': return 'deb https://example.invalid/debian/ stable main' - case 'pypi': return 'pip config set global.index-url https://example.invalid/pypi/simple' - default: return `https://example.invalid/${selectedMirror.value?.name || ''}/` - } -}) -async function copyConfig() { - copied.value = false - copyError.value = false - try { await navigator.clipboard.writeText(configExample.value); copied.value = true } - catch { copyError.value = true } -} function onKeydown(event: KeyboardEvent) { if (event.key === 'Escape') { themeOpen.value = false; mobileOpen.value = false } @@ -145,7 +127,6 @@ function outsideClick(event: MouseEvent) { function onSystemTheme(event: MediaQueryListEvent) { prefersDark.value = event.matches } let interval: ReturnType let boardInterval: ReturnType -let elapsed = 0 onMounted(() => { document.addEventListener('keydown', onKeydown) document.addEventListener('click', outsideClick) @@ -158,7 +139,6 @@ onMounted(() => { interval = setInterval(() => { if (document.visibilityState !== 'visible') return now.value = Date.now() - if (++elapsed % 9 === 0) advanceDemo() }, 1000) }) onBeforeUnmount(() => { @@ -172,9 +152,9 @@ onBeforeUnmount(() => {