feat: add configurable mirror source generators and version management

This commit is contained in:
2026-11-05 13:06:57 +08:00
parent 8fc4feb10d
commit 8ebf69bf0a
38 changed files with 1173 additions and 314 deletions
+9 -2
View File
@@ -77,7 +77,7 @@ func strict(raw []byte, value any) bool {
}
func (a *App) listRecords(w http.ResponseWriter, r *http.Request) {
kind := r.PathValue("kind")
if kind != "announcements" && kind != "mirrors" && kind != "settings" {
if kind != "announcements" && kind != "mirrors" && kind != "settings" && kind != "generators" {
fail(w, 404, "NOT_FOUND")
return
}
@@ -93,7 +93,7 @@ func (a *App) listRecords(w http.ResponseWriter, r *http.Request) {
}
func (a *App) putRecord(w http.ResponseWriter, r *http.Request) {
kind, id := r.PathValue("kind"), r.PathValue("id")
if !validName(id) || (kind != "announcements" && kind != "mirrors" && kind != "settings") {
if !validName(id) || (kind != "announcements" && kind != "mirrors" && kind != "settings" && kind != "generators") {
fail(w, 400, "INVALID_INPUT")
return
}
@@ -110,6 +110,13 @@ func (a *App) putRecord(w http.ResponseWriter, r *http.Request) {
}
var value any
switch kind {
case "generators":
var data GeneratorVersions
if !strict(body.Data, &data) || !validGeneratorVersions(id, data) {
fail(w, 400, "INVALID_INPUT")
return
}
value = data
case "announcements":
var d Announcement
if !strict(body.Data, &d) || strings.TrimSpace(d.Title) == "" || len(d.Title) > 300 || len(d.Body) > 250000 || (d.Published && strings.TrimSpace(d.Body) == "") {
+46
View File
@@ -0,0 +1,46 @@
{
"debian": {
"defaultRelease": "trixie",
"releases": [
{ "id": "trixie", "label": "Debian 13 (trixie)", "stable": true, "firmware": true, "backports": true },
{ "id": "bookworm", "label": "Debian 12 (bookworm)", "stable": true, "firmware": true, "backports": false },
{ "id": "bullseye", "label": "Debian 11 (bullseye)", "stable": true, "firmware": false, "backports": false },
{ "id": "testing", "label": "testing", "stable": false, "firmware": true, "backports": false },
{ "id": "sid", "label": "sid", "stable": false, "firmware": true, "backports": false }
]
},
"ubuntu": {
"defaultRelease": "noble",
"releases": [
{ "id": "resolute", "label": "Ubuntu 26.04 LTS (resolute)" },
{ "id": "noble", "label": "Ubuntu 24.04 LTS (noble)" },
{ "id": "jammy", "label": "Ubuntu 22.04 LTS (jammy)" }
]
},
"kubernetes": {
"defaultRelease": "v1.35",
"releases": [
{ "id": "v1.37", "label": "v1.37" },
{ "id": "v1.36", "label": "v1.36" },
{ "id": "v1.35", "label": "v1.35" },
{ "id": "v1.34", "label": "v1.34" }
]
},
"tailscale": {
"defaultRelease": "debian/bookworm",
"releases": [
{ "id": "debian/trixie", "label": "Debian 13 (trixie)" },
{ "id": "debian/bookworm", "label": "Debian 12 (bookworm)" },
{ "id": "ubuntu/resolute", "label": "Ubuntu 26.04 LTS (resolute)" },
{ "id": "ubuntu/noble", "label": "Ubuntu 24.04 LTS (noble)" },
{ "id": "ubuntu/jammy", "label": "Ubuntu 22.04 LTS (jammy)" }
]
},
"proxmox": {
"defaultRelease": "trixie",
"releases": [
{ "id": "trixie", "label": "Proxmox VE 9 / Debian 13 (trixie)", "keyring": "/usr/share/keyrings/proxmox-archive-keyring.gpg" },
{ "id": "bookworm", "label": "Proxmox VE 8 / Debian 12 (bookworm)", "keyring": "/etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg" }
]
}
}
+123
View File
@@ -0,0 +1,123 @@
package main
import (
_ "embed"
"encoding/json"
"errors"
"net/http"
"regexp"
"strings"
"unicode"
)
//go:embed generator-defaults.json
var generatorDefaultsJSON []byte
type GeneratorRelease struct {
ID string `json:"id"`
Label string `json:"label"`
Stable *bool `json:"stable,omitempty"`
Firmware *bool `json:"firmware,omitempty"`
Backports *bool `json:"backports,omitempty"`
Keyring string `json:"keyring,omitempty"`
}
type GeneratorVersions struct {
DefaultRelease string `json:"defaultRelease"`
Releases []GeneratorRelease `json:"releases"`
}
var generatorTypes = []string{"debian", "ubuntu", "kubernetes", "tailscale", "proxmox"}
var suiteID = regexp.MustCompile(`^[a-z][a-z0-9-]{0,31}$`)
var kubernetesID = regexp.MustCompile(`^v[1-9][0-9]{0,2}\.(0|[1-9][0-9]{0,2})$`)
var tailscaleID = regexp.MustCompile(`^(debian|ubuntu)/[a-z][a-z0-9-]{0,31}$`)
var keyringPath = regexp.MustCompile(`^/(usr/share/keyrings|etc/apt/trusted\.gpg\.d)/[a-zA-Z0-9][a-zA-Z0-9_-]{0,95}\.gpg$`)
func generatorType(id string) bool {
for _, known := range generatorTypes {
if id == known {
return true
}
}
return false
}
// Values are inserted into repository files and shell commands. Accept only
// version identifiers and keyring paths, never arbitrary templates or scripts.
func validGeneratorVersions(id string, data GeneratorVersions) bool {
if !generatorType(id) || len(data.Releases) == 0 || len(data.Releases) > 100 {
return false
}
seen := map[string]bool{}
hasDefault, hasStable := false, false
for _, release := range data.Releases {
validID := suiteID.MatchString(release.ID)
if id == "kubernetes" {
validID = kubernetesID.MatchString(release.ID)
}
if id == "tailscale" {
validID = tailscaleID.MatchString(release.ID)
}
if !validID || seen[release.ID] || strings.TrimSpace(release.Label) == "" || len([]rune(release.Label)) > 160 || strings.IndexFunc(release.Label, unicode.IsControl) >= 0 {
return false
}
seen[release.ID] = true
hasDefault = hasDefault || release.ID == data.DefaultRelease
if id == "debian" {
if release.Stable == nil || release.Firmware == nil || release.Backports == nil || (*release.Backports && !*release.Stable) {
return false
}
hasStable = hasStable || *release.Stable
} else if release.Stable != nil || release.Firmware != nil || release.Backports != nil {
return false
}
if id == "proxmox" {
if !keyringPath.MatchString(release.Keyring) {
return false
}
} else if release.Keyring != "" {
return false
}
}
return hasDefault && (id != "debian" || hasStable)
}
// Seed only missing records. Subsequent releases never overwrite admin edits.
func (a *App) seedGeneratorVersions() error {
var defaults map[string]GeneratorVersions
if err := json.Unmarshal(generatorDefaultsJSON, &defaults); err != nil {
return err
}
for _, id := range generatorTypes {
data := defaults[id]
if !validGeneratorVersions(id, data) {
return errors.New("invalid generator defaults: " + id)
}
if _, err := a.record("generators", id); errors.Is(err, errMissing) {
if _, err = a.saveRecord("generators", id, data, 0); err != nil {
return err
}
} else if err != nil {
return err
}
}
return nil
}
func (a *App) generatorCatalog(w http.ResponseWriter, r *http.Request) {
records, err := a.records("generators")
if err != nil {
a.internal(w, err)
return
}
catalog := map[string]GeneratorVersions{}
for _, record := range records {
var data GeneratorVersions
if decodeData(record.Data, &data) != nil || !validGeneratorVersions(record.ID, data) {
a.internal(w, errors.New("invalid generator record: "+record.ID))
return
}
catalog[record.ID] = data
}
respond(w, 200, catalog)
}
+109
View File
@@ -0,0 +1,109 @@
package main
import (
"encoding/json"
"reflect"
"testing"
)
func TestGeneratorCatalogPersistenceAndAuthorization(t *testing.T) {
c := fixture(t)
w := c.call("GET", "/api/v1/generators", nil, false)
status(t, w, 200)
var catalog map[string]GeneratorVersions
json.Unmarshal(w.Body.Bytes(), &catalog)
if len(catalog) != 5 {
t.Fatal("missing seeded version catalogs")
}
settingsBefore, _ := c.app.record("settings", "site")
before, _ := c.app.record("generators", "ubuntu")
versions := catalog["ubuntu"]
// Deliberately fictional future data: no official codename is assumed.
versions.Releases = append(versions.Releases, GeneratorRelease{ID: "future-test", Label: "Ubuntu future (test fixture)"})
versions.DefaultRelease = "future-test"
body := map[string]any{"data": versions, "revision": before.Revision}
path := "/api/v1/admin/records/generators/ubuntu"
status(t, c.call("PUT", path, body, true), 401)
c.login("editor")
status(t, c.call("GET", "/api/v1/admin/records/generators", nil, false), 403)
status(t, c.call("PUT", path, body, true), 403)
c.login("admin")
status(t, c.call("PUT", path, body, false), 403)
status(t, c.call("PUT", path, body, true), 200)
status(t, c.call("PUT", path, body, true), 409)
status(t, c.call("DELETE", path, map[string]any{"revision": before.Revision + 1}, true), 400)
// Starting a new app on the same DB must not overwrite edited defaults.
c.app.db.Close()
restarted, err := newApp(c.app.cfg)
if err != nil {
t.Fatal(err)
}
c.app = restarted
t.Cleanup(func() { restarted.db.Close() })
w = c.call("GET", "/api/v1/generators", nil, false)
status(t, w, 200)
json.Unmarshal(w.Body.Bytes(), &catalog)
if !reflect.DeepEqual(catalog["ubuntu"], versions) {
t.Fatal("saved future release lost after restart")
}
settingsAfter, _ := c.app.record("settings", "site")
if !reflect.DeepEqual(settingsBefore, settingsAfter) {
t.Fatal("generator edits changed site settings")
}
var records []Record
w = c.call("GET", "/api/v1/admin/records/generators", nil, false)
status(t, w, 200)
json.Unmarshal(w.Body.Bytes(), &records)
if len(records) != 5 {
t.Fatal("admin cannot read saved version catalogs")
}
}
func TestGeneratorCatalogRejectsUnsafeAndInconsistentData(t *testing.T) {
c := fixture(t)
c.login("admin")
var defaults map[string]GeneratorVersions
json.Unmarshal(generatorDefaultsJSON, &defaults)
for _, id := range generatorTypes {
if !validGeneratorVersions(id, defaults[id]) {
t.Fatal("invalid shared seed", id)
}
}
for _, id := range []string{"ubuntu", "kubernetes", "tailscale", "proxmox", "debian"} {
for _, bad := range []string{"$(id)", "noble\ntrusted=yes", "../../etc/passwd", "foo;id", "foo bar", "__proto__"} {
data := GeneratorVersions{DefaultRelease: bad, Releases: []GeneratorRelease{{ID: bad, Label: "test"}}}
if validGeneratorVersions(id, data) {
t.Fatal("accepted unsafe ID", id, bad)
}
}
}
invalid := []any{
map[string]any{"defaultRelease": "noble", "releases": []any{}},
map[string]any{"defaultRelease": "missing", "releases": []GeneratorRelease{{ID: "noble", Label: "Ubuntu"}}},
map[string]any{"defaultRelease": "noble", "releases": []GeneratorRelease{{ID: "noble", Label: "Ubuntu"}, {ID: "noble", Label: "duplicate"}}},
map[string]any{"defaultRelease": "noble", "releases": []GeneratorRelease{{ID: "noble", Label: ""}}},
map[string]any{"defaultRelease": "noble", "releases": []any{map[string]any{"id": "noble", "label": "Ubuntu", "command": "id"}}},
map[string]any{"defaultRelease": "noble", "releases": []GeneratorRelease{{ID: "noble", Label: "Ubuntu", Keyring: "/tmp/other.gpg"}}},
}
record, _ := c.app.record("generators", "ubuntu")
for _, data := range invalid {
status(t, c.call("PUT", "/api/v1/admin/records/generators/ubuntu", map[string]any{"data": data, "revision": record.Revision}, true), 400)
}
status(t, c.call("PUT", "/api/v1/admin/records/generators/unknown", map[string]any{"data": defaults["ubuntu"], "revision": 0}, true), 400)
pve := defaults["proxmox"]
for _, path := range []string{"/tmp/a.gpg", "/usr/share/keyrings/../../tmp/a.gpg", "/usr/share/keyrings/a.gpg\nmalicious", "/usr/share/keyrings/$(id).gpg"} {
pve.Releases[0].Keyring = path
if validGeneratorVersions("proxmox", pve) {
t.Fatal("accepted unsafe keyring", path)
}
}
debian := defaults["debian"]
for index := range debian.Releases {
value := false
debian.Releases[index].Stable = &value
debian.Releases[index].Backports = &value
}
if validGeneratorVersions("debian", debian) {
t.Fatal("Debian Security lost all stable releases")
}
}
+5
View File
@@ -91,6 +91,10 @@ func newApp(cfg Config) (*App, error) {
db.Close()
return nil, err
}
if err = a.seedGeneratorVersions(); err != nil {
db.Close()
return nil, err
}
return a, nil
}
func (a *App) handler() http.Handler {
@@ -105,6 +109,7 @@ func (a *App) handler() http.Handler {
mux.HandleFunc("GET /api/v1/public", a.public)
mux.HandleFunc("GET /api/v1/announcements", a.announcements)
mux.HandleFunc("GET /api/v1/help", a.helpIndex)
mux.HandleFunc("GET /api/v1/generators", a.generatorCatalog)
mux.HandleFunc("GET /api/v1/help/{name}", a.helpArticle)
mux.HandleFunc("GET /api/v1/monitoring", a.monitoring)
mux.HandleFunc("GET /api/v1/job-speeds", a.jobSpeeds)