feat(trainer): training from the collector UI — the trainer becomes a job service, the review page gains a Training section

Trainer: `parking-trainer serve` — a stdlib HTTP job API on the compose network (never
published): /health, /readiness, /versions, /versions/<v>/report, /jobs. One job at a
time; each job runs the CLI as a subprocess with its output captured, state + log
persisted under /out/jobs/ so a restart keeps history. `publish` takes its URL from
TRAINER_PUBLISH_URL. Dockerfile: CMD serve, EXPOSE 8091, healthcheck.

Collector: COLLECTOR_TRAINER_URL + /api/training/{status,jobs,jobs/:id,versions/:v/report}
— a reviewer-gated proxy that forwards a fixed set of paths and whitelisted knobs and
passes the trainer's status codes through (409 while a job runs; 503 unconfigured, 502
unreachable). /review gains the Training section: labels per class vs the minimum with
Train disabled until two classes clear it, mode / backbone / floor, the running job's
live log, the versions with Report / Evaluate / Publish (publish confirms), and the
reminder that pinning stays a git commit. Fixed on the way: an apostrophe in the page's
inline script broke the whole page — a test now parses the script.

Compose: `trainer` is a service (restart: unless-stopped, read-only data volume, its own
trainer-out volume), the `train` profile and TRAINER_OUT are gone; the Docker-socket
route was rejected (root on the host for a service booths upload to). Verified with both
images running together: a Train started through the proxy finished, version and report
came back, the page rendered.

Wiki: bodytype-classifier-training (loop, running it, operating notes superseded),
vision-review-outbox, fleet-deployment-komodo, log.

Claude-Session: https://claude.ai/code/session_01FWncR69HgGPuei1dLrW3cU
This commit is contained in:
2026-09-07 14:34:13 +02:00
parent 3e77a4ad7c
commit 4ff31557a8
17 changed files with 915 additions and 75 deletions
+1 -1
View File
@@ -16,7 +16,7 @@ const basic = "Basic " + Buffer.from(`${REVIEWER.user}:${REVIEWER.pass}`).toStri
beforeEach(async () => {
dir = await mkdtemp(path.join(tmpdir(), "collector-"));
app = await buildCollector({ host: "127.0.0.1", port: 0, dataDir: dir, boothTokens: TOKENS, reviewer: REVIEWER }, { dbFile: ":memory:" });
app = await buildCollector({ host: "127.0.0.1", port: 0, dataDir: dir, boothTokens: TOKENS, reviewer: REVIEWER, trainerUrl: null }, { dbFile: ":memory:" });
await app.ready();
});
afterEach(async () => {
+51
View File
@@ -15,6 +15,8 @@ import { reviewPage } from "./review-page.js";
// /review + /api/* the reviewer's screen (HTTP Basic, one login)
// GET /export/labels.csv the training set: reviewed, usable rows (crops sit beside it on
// the volume, so the trainer on this host reads them directly)
// /api/training/* the Training section: a thin proxy to the trainer's job API on
// the compose network (never published), behind the reviewer login
// It deliberately has no fleet features and no path back into a booth.
/** The package's `meta` part, as the booth sends it (review-outbox.ts). */
@@ -230,6 +232,55 @@ export async function buildCollector(cfg: CollectorConfig, opts: { dbFile?: stri
return reply.type("text/csv; charset=utf-8").header("content-disposition", 'attachment; filename="labels.csv"').send([head, ...lines].join("\n") + "\n");
});
// --- Training (proxy to the trainer's job API) ----------------------------------------
// The trainer is a sibling container reading the same volume; it is reachable only on the
// compose network, so the reviewer's login here is the only gate. The proxy forwards a
// fixed set of paths and passes the trainer's status codes through (409 = a job runs).
const trainer = cfg.trainerUrl;
async function viaTrainer(reply: FastifyReply, tpath: string, init?: RequestInit): Promise<unknown> {
if (!trainer) return reply.code(503).send({ error: "trainer not configured" });
let r: Response;
try {
r = await fetch(trainer + tpath, { ...init, signal: AbortSignal.timeout(15_000) });
} catch (err) {
return reply.code(502).send({ error: `trainer unreachable: ${(err as Error).message}` });
}
const ctype = r.headers.get("content-type") ?? "application/json";
return reply.code(r.status).type(ctype).send(Buffer.from(await r.arrayBuffer()));
}
app.get("/api/training/status", { preHandler: requireReviewer }, async (_req, reply) => {
if (!trainer) return { configured: false };
try {
const get = async (p: string) => {
const r = await fetch(trainer + p, { signal: AbortSignal.timeout(15_000) });
if (!r.ok) throw new Error(`${p} → HTTP ${r.status}`);
return r.json() as Promise<Record<string, unknown>>;
};
const [health, readiness, versions, jobs] = await Promise.all([get("/health"), get("/readiness"), get("/versions"), get("/jobs")]);
return { configured: true, reachable: true, health, readiness, versions: versions.versions, jobs: jobs.jobs, current: jobs.current };
} catch (err) {
return reply.code(200).send({ configured: true, reachable: false, error: (err as Error).message });
}
});
app.post<{ Body: Record<string, unknown> }>("/api/training/jobs", { preHandler: requireReviewer }, async (req, reply) => {
const b = req.body && typeof req.body === "object" ? req.body : {};
const kind = b.kind;
if (kind !== "train" && kind !== "evaluate" && kind !== "publish") return reply.code(400).send({ error: "kind must be train, evaluate or publish" });
// Only the knobs the UI offers cross over; the trainer validates their values.
const allowed = ["kind", "mode", "backbone", "minAccuracy", "minPerClass", "epochs", "version"];
const body: Record<string, unknown> = {};
for (const k of allowed) if (b[k] !== undefined) body[k] = b[k];
return viaTrainer(reply, "/jobs", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
});
app.get<{ Params: { id: string } }>("/api/training/jobs/:id", { preHandler: requireReviewer }, async (req, reply) => {
if (!ID_RE.test(req.params.id)) return reply.code(400).send({ error: "bad job id" });
return viaTrainer(reply, `/jobs/${encodeURIComponent(req.params.id)}`);
});
app.get<{ Params: { v: string } }>("/api/training/versions/:v/report", { preHandler: requireReviewer }, async (req, reply) => {
if (!ID_RE.test(req.params.v)) return reply.code(400).send({ error: "bad version" });
return viaTrainer(reply, `/versions/${encodeURIComponent(req.params.v)}/report`);
});
return app;
}
+4
View File
@@ -6,6 +6,9 @@ export interface CollectorConfig {
readonly boothTokens: ReadonlyMap<string, string>;
/** The single reviewer login; null = review screen and export refuse (503). */
readonly reviewer: { readonly user: string; readonly pass: string } | null;
/** The trainer's job API on the compose network (http://trainer:8091); null = the
* Training section is hidden and /api/training/* answers 503. */
readonly trainerUrl: string | null;
}
/** "booth-7:abc,booth-9:def" (commas, whitespace or newlines between pairs). */
@@ -32,5 +35,6 @@ export function configFromEnv(env: NodeJS.ProcessEnv = process.env): CollectorCo
dataDir: env.COLLECTOR_DATA_DIR ?? "/data",
boothTokens: parseBoothTokens(env.COLLECTOR_BOOTH_TOKENS ?? ""),
reviewer: user && pass.length >= 8 ? { user, pass } : null,
trainerUrl: (env.COLLECTOR_TRAINER_URL ?? "").trim().replace(/\/+$/, "") || null,
};
}
+1 -1
View File
@@ -5,7 +5,7 @@ const cfg = configFromEnv();
const app = await buildCollector(cfg);
if (cfg.boothTokens.size === 0) app.log.warn("COLLECTOR_BOOTH_TOKENS is empty — no booth can ingest");
if (!cfg.reviewer) app.log.warn("COLLECTOR_REVIEWER_USER/PASS not set — the review screen and export refuse");
app.log.info(`collector: ${cfg.boothTokens.size} booth token(s), data in ${cfg.dataDir}`);
app.log.info(`collector: ${cfg.boothTokens.size} booth token(s), data in ${cfg.dataDir}, trainer ${cfg.trainerUrl ?? "not configured"}`);
await app.listen({ host: cfg.host, port: cfg.port });
const stop = async () => {
+114
View File
@@ -35,6 +35,16 @@ export function reviewPage(): string {
td, th { text-align:left; padding:.2rem .5rem; border-bottom:1px solid #2a2a2a; }
th { color:var(--muted); font-weight:normal; font-size:.75rem; text-transform:uppercase; letter-spacing:.06em; }
kbd { background:#2a2a2a; border:1px solid #444; border-radius:3px; padding:0 .3rem; font-size:.75rem; }
h2 { font-size:.8rem; letter-spacing:.08em; text-transform:uppercase; color:var(--amber); margin:0 0 .6rem; }
.row { display:flex; flex-wrap:wrap; gap:.6rem; align-items:center; }
select, input { background:#2a2a2a; color:var(--text); border:1px solid #444; border-radius:4px; padding:.4rem .5rem; font:inherit; }
input[type=number] { width:5rem; }
label { color:var(--muted); font-size:.8rem; }
pre { background:#0d0d0d; border:1px solid #2a2a2a; border-radius:4px; padding:.6rem; max-height:22rem; overflow:auto; font-size:.75rem; white-space:pre-wrap; margin:.6rem 0 0; }
.ok { color:var(--green); }
.bad { color:var(--red); }
button:disabled { opacity:.45; cursor:not-allowed; }
button.small { padding:.25rem .5rem; font-size:.75rem; }
</style>
</head>
<body>
@@ -46,6 +56,10 @@ export function reviewPage(): string {
<section class="card">
<table id="stats"><thead><tr><th>booth</th><th>operator</th><th>reviewed</th><th>agree</th><th>disagree</th><th>unusable</th></tr></thead><tbody></tbody></table>
</section>
<section class="card" id="training" hidden>
<h2>Training</h2>
<div id="tr-body"></div>
</section>
<p class="muted">Keys: <kbd>1</kbd>–<kbd>9</kbd>, <kbd>0</kbd> pick a class in order · <kbd>u</kbd> unusable · <kbd>s</kbd> skip. Skipped items come back after a reload. Your verdict is the training label; the operator's pick is only compared against it.</p>
</main>
<script>
@@ -114,6 +128,106 @@ document.addEventListener('keydown', e => {
next().catch(e => { document.getElementById('item').innerHTML = '<p class="warn">' + esc(e.message) + '</p>'; });
loadStats().catch(() => {});
// ---- Training: the trainer's job API, proxied by the collector -------------------------
// Readiness (labels per class vs the minimum), one job at a time with a live log, the
// versions a run produced (written or refused) with Report / Evaluate / Publish. Pinning a
// published version into the vision image stays a git commit — that is the deploy control.
let trPoll = null;
let trShownReport = null;
const trDefaults = { mode: 'features', backbone: 'resnet18', minAccuracy: 0.85 };
function pct(x) { return x == null ? '—' : Math.round(x * 100) + ' %'; }
async function training() {
const box = document.getElementById('training');
const el = document.getElementById('tr-body');
let s;
try { s = await api('/api/training/status'); } catch (e) { box.hidden = false; el.innerHTML = '<p class="warn">' + esc(e.message) + '</p>'; return; }
if (!s.configured) { box.hidden = true; return; }
box.hidden = false;
if (!s.reachable) { el.innerHTML = '<p class="warn">trainer not reachable: ' + esc(s.error || '') + '</p>'; schedule(true); return; }
const r = s.readiness, run = r.run || {}, minPer = run.minPerClass || 20;
const byClass = (r.labelled && r.labelled.byClass) || {};
const classes = Object.keys(byClass);
const cur = s.current;
const readyLine = r.ready
? '<span class="ok">enough labels to train</span> — classes this run: ' + esc((run.classes || []).join(', '))
: '<span class="warn">not enough labels yet</span> — a class needs ' + minPer + ' reviewed crops; two classes must clear it';
let html = '<p>' + readyLine + ' <span class="muted">(' + (r.labelled ? r.labelled.total : 0) + ' labelled, ' + (r.missingCrops || 0) + ' missing crop files)</span></p>';
html += '<table><thead><tr><th>class</th><th>reviewed</th><th>train</th><th>val</th><th></th></tr></thead><tbody>' +
(classes.map(c => '<tr><td class="mono">' + esc(c) + '</td><td>' + byClass[c] + '</td><td>' + ((run.train || {})[c] ?? '—') + '</td><td>' + ((run.val || {})[c] ?? '—') + '</td><td class="muted">' + (byClass[c] < minPer ? 'below ' + minPer + ' — dropped' : '') + '</td></tr>').join('') || '<tr><td colspan="5" class="muted">no labels yet — review crops above</td></tr>') +
'</tbody></table>';
const d = Object.assign({}, trDefaults, r.defaults || {});
html += '<div class="row" style="margin-top:.8rem">' +
'<label>mode <select id="tr-mode">' + (r.modes || ['features', 'finetune']).map(m => '<option' + (m === d.mode ? ' selected' : '') + '>' + m + '</option>').join('') + '</select></label>' +
'<label>backbone <select id="tr-backbone">' + (r.backbones || ['resnet18']).map(b => '<option' + (b === d.backbone ? ' selected' : '') + '>' + b + '</option>').join('') + '</select></label>' +
'<label>floor <input id="tr-floor" type="number" min="0" max="1" step="0.01" value="' + d.minAccuracy + '"></label>' +
'<button id="tr-train"' + (r.ready && !cur ? '' : ' disabled') + '>Train</button>' +
(cur ? '<span class="warn">running: ' + esc(cur.kind) + ' ' + esc(cur.id) + '</span>' : '') +
'</div>';
const last = cur || (s.jobs && s.jobs[0]);
if (last) {
const cls = last.status === 'done' ? 'ok' : last.status === 'running' ? 'warn' : 'bad';
html += '<p style="margin:.8rem 0 0"><span class="' + cls + '">' + esc(last.status) + '</span> <span class="mono">' + esc(last.kind) + ' ' + esc(last.id) + '</span> <span class="muted">' + esc(last.startedAt || '') + (last.exitCode != null ? ' · exit ' + last.exitCode : '') + '</span> <button class="small" data-job="' + esc(last.id) + '">log</button></p>' +
'<pre id="tr-log" hidden></pre>';
}
const vs = s.versions || [];
html += '<h2 style="margin-top:1rem">Versions</h2>';
html += vs.length
? '<table><thead><tr><th>version</th><th>model</th><th>accuracy</th><th>classes</th><th>mode</th><th></th></tr></thead><tbody>' +
vs.map(v => '<tr><td class="mono">' + esc(v.version) + '</td><td>' + (v.written ? '<span class="ok">written</span>' : '<span class="bad">refused</span>') + '</td><td>' + pct(v.accuracy) + (v.floor != null ? ' <span class="muted">/ floor ' + pct(v.floor) + '</span>' : '') + '</td><td class="muted">' + esc((v.classes || []).join(', ')) + '</td><td class="muted">' + esc(v.mode || '') + '</td><td>' +
'<button class="small" data-report="' + esc(v.version) + '">report</button> ' +
(v.written ? '<button class="small" data-eval="' + esc(v.version) + '"' + (cur ? ' disabled' : '') + '>evaluate</button> <button class="small" data-publish="' + esc(v.version) + '"' + (cur ? ' disabled' : '') + '>publish</button>' : '') +
'</td></tr>').join('') + '</tbody></table>'
: '<p class="muted">no runs yet</p>';
html += '<pre id="tr-report" hidden></pre>';
html += '<p class="muted" style="margin:.8rem 0 0">A written model is only a file here. To put it on a booth: publish, then pin the version in <span class="mono">apps/vision/models/bodytype.version</span>, commit, and bump the TAG of the booth.</p>';
el.innerHTML = html;
const trainBtn = document.getElementById('tr-train');
if (trainBtn) trainBtn.addEventListener('click', () => startJob({ kind: 'train', mode: document.getElementById('tr-mode').value, backbone: document.getElementById('tr-backbone').value, minAccuracy: Number(document.getElementById('tr-floor').value) }));
el.querySelectorAll('button[data-eval]').forEach(b => b.addEventListener('click', () => startJob({ kind: 'evaluate', version: b.dataset.eval })));
el.querySelectorAll('button[data-publish]').forEach(b => b.addEventListener('click', () => { if (confirm('Publish ' + b.dataset.publish + ' to the package registry?')) startJob({ kind: 'publish', version: b.dataset.publish }); }));
el.querySelectorAll('button[data-job]').forEach(b => b.addEventListener('click', () => showLog(b.dataset.job)));
el.querySelectorAll('button[data-report]').forEach(b => b.addEventListener('click', () => showReport(b.dataset.report)));
if (cur) showLog(cur.id).catch(() => {});
if (trShownReport) showReport(trShownReport).catch(() => {});
schedule(!!cur);
}
function schedule(soon) {
if (trPoll) clearTimeout(trPoll);
trPoll = setTimeout(() => training().catch(() => {}), soon ? 4000 : 60000);
}
async function startJob(body) {
try {
const r = await fetch('/api/training/jobs', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });
if (!r.ok) { const e = await r.json().catch(() => ({})); alert('trainer: ' + (e.error || ('HTTP ' + r.status))); }
} catch (e) { alert(e.message); }
training().catch(() => {});
}
async function showLog(id) {
const j = await api('/api/training/jobs/' + encodeURIComponent(id));
const pre = document.getElementById('tr-log');
if (!pre) return;
pre.hidden = false;
pre.textContent = j.log || '(no output yet)';
pre.scrollTop = pre.scrollHeight;
}
async function showReport(v) {
const r = await fetch('/api/training/versions/' + encodeURIComponent(v) + '/report');
const pre = document.getElementById('tr-report');
if (!pre) return;
trShownReport = v;
pre.hidden = false;
pre.textContent = r.ok ? await r.text() : 'no report for ' + v + ' (HTTP ' + r.status + ')';
}
training().catch(() => {});
</script>
</body>
</html>`;
+126
View File
@@ -0,0 +1,126 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { buildCollector, type CollectorApp } from "./app.js";
// The Training section's proxy: reviewer-gated, forwards a fixed set of paths to the
// trainer's job API, passes its status codes through, and degrades cleanly when the trainer
// is not configured or not reachable. The trainer is faked with a bare node http server.
const REVIEWER = { user: "julian", pass: "review-pass-123" };
const basic = "Basic " + Buffer.from(`${REVIEWER.user}:${REVIEWER.pass}`).toString("base64");
let dir: string;
let fake: Server;
let fakeUrl: string;
let seen: { method: string; url: string; body: string }[];
let app: CollectorApp;
async function start(trainerUrl: string | null): Promise<void> {
app = await buildCollector({ host: "127.0.0.1", port: 0, dataDir: dir, boothTokens: new Map(), reviewer: REVIEWER, trainerUrl }, { dbFile: ":memory:" });
await app.ready();
}
beforeEach(async () => {
dir = await mkdtemp(path.join(tmpdir(), "collector-"));
seen = [];
fake = createServer((req: IncomingMessage, res: ServerResponse) => {
let body = "";
req.on("data", (c) => (body += c));
req.on("end", () => {
seen.push({ method: req.method ?? "", url: req.url ?? "", body });
const json = (code: number, obj: unknown) => {
res.writeHead(code, { "content-type": "application/json" });
res.end(JSON.stringify(obj));
};
if (req.url === "/health") return json(200, { ok: true, busy: false });
if (req.url === "/readiness") return json(200, { ready: false, labelled: { total: 3 } });
if (req.url === "/versions") return json(200, { versions: [{ version: "v1", written: true }] });
if (req.url === "/jobs" && req.method === "GET") return json(200, { jobs: [{ id: "j1" }], current: null });
if (req.url === "/jobs" && req.method === "POST") return body.includes('"busy"') ? json(409, { error: "a job is already running" }) : json(202, { id: "j2", status: "running" });
if (req.url === "/jobs/j1") return json(200, { id: "j1", status: "done", log: "ok" });
if (req.url === "/versions/v1/report") {
res.writeHead(200, { "content-type": "text/markdown; charset=utf-8" });
return res.end("# Body-type classifier v1\n");
}
return json(404, { error: "not found" });
});
});
await new Promise<void>((r) => fake.listen(0, "127.0.0.1", r));
const a = fake.address() as { port: number };
fakeUrl = `http://127.0.0.1:${a.port}`;
});
afterEach(async () => {
await app?.close();
await new Promise<void>((r) => fake.close(() => r()));
await rm(dir, { recursive: true, force: true });
});
describe("review page script", () => {
it("parses as JavaScript (an apostrophe in a template literal once broke the whole page)", async () => {
const { reviewPage } = await import("./review-page.js");
const html = reviewPage();
const script = html.slice(html.indexOf("<script>") + 8, html.lastIndexOf("</script>"));
expect(() => new Function(script)).not.toThrow();
});
});
describe("training proxy", () => {
it("is hidden when no trainer is configured", async () => {
await start(null);
const s = await app.inject({ method: "GET", url: "/api/training/status", headers: { authorization: basic } });
expect(s.json()).toEqual({ configured: false });
const j = await app.inject({ method: "POST", url: "/api/training/jobs", headers: { authorization: basic }, payload: { kind: "train" } });
expect(j.statusCode).toBe(503);
});
it("aggregates status and forwards jobs and reports behind the reviewer login", async () => {
await start(fakeUrl);
expect((await app.inject({ method: "GET", url: "/api/training/status" })).statusCode).toBe(401);
const s = await app.inject({ method: "GET", url: "/api/training/status", headers: { authorization: basic } });
expect(s.statusCode).toBe(200);
const body = s.json();
expect(body.configured).toBe(true);
expect(body.reachable).toBe(true);
expect(body.readiness.labelled.total).toBe(3);
expect(body.versions[0].version).toBe("v1");
expect(body.jobs[0].id).toBe("j1");
const j = await app.inject({
method: "POST",
url: "/api/training/jobs",
headers: { authorization: basic },
payload: { kind: "train", mode: "features", minAccuracy: 0.9, secret: "nope", version: "v2" },
});
expect(j.statusCode).toBe(202);
expect(j.json().id).toBe("j2");
const posted = seen.find((r) => r.method === "POST")!;
expect(JSON.parse(posted.body)).toEqual({ kind: "train", mode: "features", minAccuracy: 0.9, version: "v2" }); // unknown keys dropped
const busy = await app.inject({ method: "POST", url: "/api/training/jobs", headers: { authorization: basic }, payload: { kind: "evaluate", version: "busy" } });
expect(busy.statusCode).toBe(409); // the trainer's answer passes through
const bad = await app.inject({ method: "POST", url: "/api/training/jobs", headers: { authorization: basic }, payload: { kind: "rm-rf" } });
expect(bad.statusCode).toBe(400);
const one = await app.inject({ method: "GET", url: "/api/training/jobs/j1", headers: { authorization: basic } });
expect(one.json().status).toBe("done");
expect((await app.inject({ method: "GET", url: "/api/training/jobs/..%2Fx", headers: { authorization: basic } })).statusCode).toBe(400);
const rep = await app.inject({ method: "GET", url: "/api/training/versions/v1/report", headers: { authorization: basic } });
expect(rep.statusCode).toBe(200);
expect(rep.headers["content-type"]).toContain("text/markdown");
expect(rep.body).toContain("# Body-type classifier v1");
});
it("reports an unreachable trainer without failing the page", async () => {
await start("http://127.0.0.1:9"); // nothing listens on the discard port
const s = await app.inject({ method: "GET", url: "/api/training/status", headers: { authorization: basic } });
expect(s.statusCode).toBe(200);
expect(s.json().reachable).toBe(false);
const j = await app.inject({ method: "POST", url: "/api/training/jobs", headers: { authorization: basic }, payload: { kind: "train" } });
expect(j.statusCode).toBe(502);
});
});