Compare commits
21 Commits
8bfc29db2a
...
stage
| Author | SHA1 | Date | |
|---|---|---|---|
| 5afdcc78ae | |||
| e3de7800e5 | |||
| 73bfc1d9b9 | |||
| f4b806a538 | |||
| fe3b12a60d | |||
| 88f9c53fda | |||
| e8cb057082 | |||
| 4a02e5fed3 | |||
| 552d87d75b | |||
| 7e21cf057e | |||
| 4fd175e0e4 | |||
| 535244209a | |||
| 0845e87ddd | |||
| 2d9bb15d4c | |||
| 29594f8bad | |||
| 4ff31557a8 | |||
| 3e77a4ad7c | |||
| 3f16925fe0 | |||
| f7a262ac9a | |||
| f9cb973fe9 | |||
| 1a0fe59488 |
@@ -1,6 +1,6 @@
|
||||
name: Build & push images
|
||||
|
||||
# Build the SERVER (API + SPA), COLLECTOR (wash review) and VISION (ANPR) container images and push them to the
|
||||
# Build the SERVER (API + SPA), COLLECTOR (wash review), VISION (ANPR) and TRAINER (phase-B job) container images and push them to the
|
||||
# house Gitea registry, tagged by BRANCH + short SHA (branch-aware: dev→:dev, stage→:stage,
|
||||
# main→:main). Separate from ci.yml (checks-only) and release.yml (tag-only desktop bundle).
|
||||
# Mirrors the house pattern (cf. trm/processor build.yml). See
|
||||
@@ -14,6 +14,7 @@ on:
|
||||
- 'apps/web/**'
|
||||
- 'apps/vision/**'
|
||||
- 'apps/collector/**'
|
||||
- 'apps/trainer/**'
|
||||
- 'packages/**'
|
||||
- 'package.json'
|
||||
- 'pnpm-lock.yaml'
|
||||
@@ -61,6 +62,11 @@ jobs:
|
||||
working-directory: apps/vision
|
||||
run: uv sync --frozen
|
||||
|
||||
- name: Sync trainer deps
|
||||
# Light core only — NOT the `train` extra (CPU torch, ~200 MB); the torch tests skip.
|
||||
working-directory: apps/trainer
|
||||
run: uv sync --frozen
|
||||
|
||||
# Don't publish a broken image — run the same checks as ci.yml first.
|
||||
- name: Build + lint + test (Turbo)
|
||||
run: pnpm turbo run build lint test
|
||||
@@ -119,12 +125,29 @@ jobs:
|
||||
context: apps/vision
|
||||
file: apps/vision/Dockerfile
|
||||
push: true
|
||||
# The phase-B body-type classifier is fetched from the Gitea generic package registry
|
||||
# at build when apps/vision/models/bodytype.version pins a version (empty = none). The
|
||||
# registry user's credentials double as the fetch auth (BuildKit secret, never a layer).
|
||||
secrets: |
|
||||
bodytype_auth=${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }}
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/parking-vision:${{ steps.meta.outputs.branch }}
|
||||
${{ env.REGISTRY }}/parking-vision:${{ steps.meta.outputs.branch }}-${{ steps.meta.outputs.sha }}
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/parking-vision:buildcache
|
||||
cache-to: type=registry,ref=${{ env.REGISTRY }}/parking-vision:buildcache,mode=max
|
||||
|
||||
- name: Build & push TRAINER (phase-B job)
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: apps/trainer
|
||||
file: apps/trainer/Dockerfile
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/parking-trainer:${{ steps.meta.outputs.branch }}
|
||||
${{ env.REGISTRY }}/parking-trainer:${{ steps.meta.outputs.branch }}-${{ steps.meta.outputs.sha }}
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/parking-trainer:buildcache
|
||||
cache-to: type=registry,ref=${{ env.REGISTRY }}/parking-trainer:buildcache,mode=max
|
||||
|
||||
# Optional: trigger a Komodo stack redeploy (cf. trm/processor). Enable by setting the
|
||||
# KOMODO_* secrets; left guarded so it no-ops until the parking stack is wired.
|
||||
- name: Trigger Komodo redeploy
|
||||
|
||||
@@ -48,6 +48,11 @@ jobs:
|
||||
working-directory: apps/vision
|
||||
run: uv sync --frozen
|
||||
|
||||
- name: Sync trainer deps
|
||||
# Same rule: light core only, not the `train` extra (CPU torch); torch tests skip.
|
||||
working-directory: apps/trainer
|
||||
run: uv sync --frozen
|
||||
|
||||
- name: Build + lint (Turbo)
|
||||
# Covers tsc typecheck, vite build, i18n catalog type-parity (a missing sq/en
|
||||
# key fails the build), AND the vision service's ruff lint via uv.
|
||||
|
||||
@@ -16,7 +16,7 @@ const basic = "Basic " + Buffer.from(`${REVIEWER.user}:${REVIEWER.pass}`).toStri
|
||||
|
||||
beforeEach(async () => {
|
||||
dir = await mkdtemp(path.join(tmpdir(), "collector-"));
|
||||
app = await buildCollector({ host: "127.0.0.1", port: 0, dataDir: dir, boothTokens: TOKENS, reviewer: REVIEWER }, { dbFile: ":memory:" });
|
||||
app = await buildCollector({ host: "127.0.0.1", port: 0, dataDir: dir, boothTokens: TOKENS, reviewer: REVIEWER, trainerUrl: null }, { dbFile: ":memory:" });
|
||||
await app.ready();
|
||||
});
|
||||
afterEach(async () => {
|
||||
@@ -151,3 +151,46 @@ describe("config", () => {
|
||||
expect(() => parseBoothTokens("nocolon")).toThrow(/bad pair/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("schema migration", () => {
|
||||
it("opens a database created before the `kind` column and adds the missing columns, so ingest and stats work", async () => {
|
||||
// art-docker-station, 2026-09-16: the volume's DB predated `kind`; CREATE TABLE IF NOT
|
||||
// EXISTS left it alone, and /health, every ingest and the trainer's readiness failed
|
||||
// with "no such column: kind". Replay: a file with the ORIGINAL column set.
|
||||
const { default: Database } = await import("better-sqlite3");
|
||||
const file = path.join(dir, "old.sqlite");
|
||||
const old = new Database(file);
|
||||
old.exec(`CREATE TABLE items (
|
||||
id TEXT PRIMARY KEY, booth TEXT NOT NULL, order_ref TEXT NOT NULL, at TEXT NOT NULL,
|
||||
service TEXT NOT NULL, vision_class TEXT NOT NULL, vision_confidence REAL NOT NULL,
|
||||
image_width INTEGER NOT NULL, image_height INTEGER NOT NULL, plate_blurred INTEGER NOT NULL,
|
||||
image_path TEXT NOT NULL, received_at TEXT NOT NULL, review_label TEXT, reviewed_at TEXT, reviewer TEXT)`);
|
||||
old.prepare(
|
||||
"INSERT INTO items (id, booth, order_ref, at, service, vision_class, vision_confidence, image_width, image_height, plate_blurred, image_path, received_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
).run("legacy-1", "booth-7", "o-0", "2026-09-01T00:00:00.000Z", "Standard", "suv", 0.8, 100, 100, 1, "crops/legacy-1.jpg", "2026-09-01T00:00:00.000Z");
|
||||
old.close();
|
||||
|
||||
const legacy = await buildCollector({ host: "127.0.0.1", port: 0, dataDir: dir, boothTokens: TOKENS, reviewer: REVIEWER, trainerUrl: null }, { dbFile: file });
|
||||
await legacy.ready();
|
||||
try {
|
||||
const health = await legacy.inject({ method: "GET", url: "/health" });
|
||||
expect(health.statusCode).toBe(200);
|
||||
expect(health.json()).toMatchObject({ ok: true, booths: 1, pending: 1 });
|
||||
|
||||
const { body, type } = multipart({ meta: JSON.stringify(meta({ item: "item-new" })) }, JPEG);
|
||||
const r = await legacy.inject({ method: "POST", url: "/ingest", headers: { authorization: `Bearer ${TOKENS.get("booth-7")!}`, "content-type": type }, payload: body });
|
||||
expect(r.statusCode).toBe(201);
|
||||
|
||||
const stats = await legacy.inject({ method: "GET", url: "/api/stats", headers: { authorization: basic } });
|
||||
expect(stats.statusCode).toBe(200);
|
||||
|
||||
// The legacy row reads back with the defaults the new columns carry.
|
||||
const cols = new Database(file, { readonly: true }).prepare("PRAGMA table_info(items)").all() as { name: string }[];
|
||||
expect(cols.map((c) => c.name)).toEqual(expect.arrayContaining(["kind", "operator_ref", "operator_classes", "vision_category_id", "downgraded"]));
|
||||
const legacyRow = new Database(file, { readonly: true }).prepare("SELECT kind, operator_classes, downgraded FROM items WHERE id = 'legacy-1'").get() as Record<string, unknown>;
|
||||
expect(legacyRow).toEqual({ kind: "wash", operator_classes: "[]", downgraded: 0 });
|
||||
} finally {
|
||||
await legacy.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,8 @@ import { reviewPage } from "./review-page.js";
|
||||
// /review + /api/* the reviewer's screen (HTTP Basic, one login)
|
||||
// GET /export/labels.csv the training set: reviewed, usable rows (crops sit beside it on
|
||||
// the volume, so the trainer on this host reads them directly)
|
||||
// /api/training/* the Training section: a thin proxy to the trainer's job API on
|
||||
// the compose network (never published), behind the reviewer login
|
||||
// It deliberately has no fleet features and no path back into a booth.
|
||||
|
||||
/** The package's `meta` part, as the booth sends it (review-outbox.ts). */
|
||||
@@ -230,6 +232,66 @@ export async function buildCollector(cfg: CollectorConfig, opts: { dbFile?: stri
|
||||
return reply.type("text/csv; charset=utf-8").header("content-disposition", 'attachment; filename="labels.csv"').send([head, ...lines].join("\n") + "\n");
|
||||
});
|
||||
|
||||
// --- Training (proxy to the trainer's job API) ----------------------------------------
|
||||
// The trainer is a sibling container reading the same volume; it is reachable only on the
|
||||
// compose network, so the reviewer's login here is the only gate. The proxy forwards a
|
||||
// fixed set of paths and passes the trainer's status codes through (409 = a job runs).
|
||||
const trainer = cfg.trainerUrl;
|
||||
async function viaTrainer(reply: FastifyReply, tpath: string, init?: RequestInit): Promise<unknown> {
|
||||
if (!trainer) return reply.code(503).send({ error: "trainer not configured" });
|
||||
let r: Response;
|
||||
try {
|
||||
r = await fetch(trainer + tpath, { ...init, signal: AbortSignal.timeout(15_000) });
|
||||
} catch (err) {
|
||||
return reply.code(502).send({ error: `trainer unreachable: ${(err as Error).message}` });
|
||||
}
|
||||
const ctype = r.headers.get("content-type") ?? "application/json";
|
||||
return reply.code(r.status).type(ctype).send(Buffer.from(await r.arrayBuffer()));
|
||||
}
|
||||
app.get("/api/training/status", { preHandler: requireReviewer }, async (_req, reply) => {
|
||||
if (!trainer) return { configured: false };
|
||||
try {
|
||||
const get = async (p: string) => {
|
||||
const r = await fetch(trainer + p, { signal: AbortSignal.timeout(15_000) });
|
||||
if (!r.ok) {
|
||||
// Surface the trainer's own error text (its handlers answer 500 JSON), so the
|
||||
// reviewer reads "no such column: kind", not just a status code.
|
||||
const detail = await r.text().then((t) => {
|
||||
try {
|
||||
return String((JSON.parse(t) as { error?: unknown }).error ?? t);
|
||||
} catch {
|
||||
return t;
|
||||
}
|
||||
}, () => "");
|
||||
throw new Error(`${p} → HTTP ${r.status}${detail ? `: ${detail.slice(0, 300)}` : ""}`);
|
||||
}
|
||||
return r.json() as Promise<Record<string, unknown>>;
|
||||
};
|
||||
const [health, readiness, versions, jobs] = await Promise.all([get("/health"), get("/readiness"), get("/versions"), get("/jobs")]);
|
||||
return { configured: true, reachable: true, health, readiness, versions: versions.versions, jobs: jobs.jobs, current: jobs.current };
|
||||
} catch (err) {
|
||||
return reply.code(200).send({ configured: true, reachable: false, error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
app.post<{ Body: Record<string, unknown> }>("/api/training/jobs", { preHandler: requireReviewer }, async (req, reply) => {
|
||||
const b = req.body && typeof req.body === "object" ? req.body : {};
|
||||
const kind = b.kind;
|
||||
if (kind !== "train" && kind !== "evaluate" && kind !== "publish") return reply.code(400).send({ error: "kind must be train, evaluate or publish" });
|
||||
// Only the knobs the UI offers cross over; the trainer validates their values.
|
||||
const allowed = ["kind", "mode", "backbone", "minAccuracy", "minPerClass", "epochs", "version"];
|
||||
const body: Record<string, unknown> = {};
|
||||
for (const k of allowed) if (b[k] !== undefined) body[k] = b[k];
|
||||
return viaTrainer(reply, "/jobs", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
||||
});
|
||||
app.get<{ Params: { id: string } }>("/api/training/jobs/:id", { preHandler: requireReviewer }, async (req, reply) => {
|
||||
if (!ID_RE.test(req.params.id)) return reply.code(400).send({ error: "bad job id" });
|
||||
return viaTrainer(reply, `/jobs/${encodeURIComponent(req.params.id)}`);
|
||||
});
|
||||
app.get<{ Params: { v: string } }>("/api/training/versions/:v/report", { preHandler: requireReviewer }, async (req, reply) => {
|
||||
if (!ID_RE.test(req.params.v)) return reply.code(400).send({ error: "bad version" });
|
||||
return viaTrainer(reply, `/versions/${encodeURIComponent(req.params.v)}/report`);
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@ export interface CollectorConfig {
|
||||
readonly boothTokens: ReadonlyMap<string, string>;
|
||||
/** The single reviewer login; null = review screen and export refuse (503). */
|
||||
readonly reviewer: { readonly user: string; readonly pass: string } | null;
|
||||
/** The trainer's job API on the compose network (http://trainer:8091); null = the
|
||||
* Training section is hidden and /api/training/* answers 503. */
|
||||
readonly trainerUrl: string | null;
|
||||
}
|
||||
|
||||
/** "booth-7:abc,booth-9:def" (commas, whitespace or newlines between pairs). */
|
||||
@@ -32,5 +35,6 @@ export function configFromEnv(env: NodeJS.ProcessEnv = process.env): CollectorCo
|
||||
dataDir: env.COLLECTOR_DATA_DIR ?? "/data",
|
||||
boothTokens: parseBoothTokens(env.COLLECTOR_BOOTH_TOKENS ?? ""),
|
||||
reviewer: user && pass.length >= 8 ? { user, pass } : null,
|
||||
trainerUrl: (env.COLLECTOR_TRAINER_URL ?? "").trim().replace(/\/+$/, "") || null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -71,6 +71,39 @@ export class CollectorDb {
|
||||
CREATE INDEX IF NOT EXISTS items_pending ON items (reviewed_at, received_at);
|
||||
CREATE INDEX IF NOT EXISTS items_booth ON items (booth, received_at);
|
||||
`);
|
||||
this.#migrate();
|
||||
}
|
||||
|
||||
/** Columns added after the first deploy, with the DDL that adds them to an EXISTING
|
||||
* table. `CREATE TABLE IF NOT EXISTS` above only shapes a NEW database; a volume that
|
||||
* was created by an earlier build keeps its old columns, and every query naming a new
|
||||
* one then fails ("no such column: kind" — art-docker-station, 2026-09-16: the
|
||||
* collector's /health, every ingest, and the trainer's readiness all broke on a DB
|
||||
* from before `kind`). Each entry must be addable to a populated table, i.e. nullable
|
||||
* or carrying a DEFAULT. Append here whenever a column joins the CREATE above. */
|
||||
static readonly #ADDED_COLUMNS: ReadonlyArray<readonly [name: string, ddl: string]> = [
|
||||
["kind", "TEXT NOT NULL DEFAULT 'wash'"],
|
||||
["operator_ref", "TEXT NOT NULL DEFAULT ''"],
|
||||
["operator_category_id", "TEXT NOT NULL DEFAULT ''"],
|
||||
["operator_category_name", "TEXT NOT NULL DEFAULT ''"],
|
||||
["operator_classes", "TEXT NOT NULL DEFAULT '[]'"],
|
||||
["vision_category_id", "TEXT"],
|
||||
["downgraded", "INTEGER NOT NULL DEFAULT 0"],
|
||||
];
|
||||
|
||||
/** Bring an existing `items` table up to the current column set (idempotent). */
|
||||
#migrate(): void {
|
||||
const present = new Set(
|
||||
(this.#db.prepare("PRAGMA table_info(items)").all() as { name: string }[]).map((c) => c.name),
|
||||
);
|
||||
for (const [name, ddl] of CollectorDb.#ADDED_COLUMNS) {
|
||||
if (!present.has(name)) this.#db.exec(`ALTER TABLE items ADD COLUMN ${name} ${ddl}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** The current column names of `items` (for tests and diagnostics). */
|
||||
columns(): string[] {
|
||||
return (this.#db.prepare("PRAGMA table_info(items)").all() as { name: string }[]).map((c) => c.name);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
|
||||
@@ -5,7 +5,7 @@ const cfg = configFromEnv();
|
||||
const app = await buildCollector(cfg);
|
||||
if (cfg.boothTokens.size === 0) app.log.warn("COLLECTOR_BOOTH_TOKENS is empty — no booth can ingest");
|
||||
if (!cfg.reviewer) app.log.warn("COLLECTOR_REVIEWER_USER/PASS not set — the review screen and export refuse");
|
||||
app.log.info(`collector: ${cfg.boothTokens.size} booth token(s), data in ${cfg.dataDir}`);
|
||||
app.log.info(`collector: ${cfg.boothTokens.size} booth token(s), data in ${cfg.dataDir}, trainer ${cfg.trainerUrl ?? "not configured"}`);
|
||||
await app.listen({ host: cfg.host, port: cfg.port });
|
||||
|
||||
const stop = async () => {
|
||||
|
||||
@@ -35,6 +35,16 @@ export function reviewPage(): string {
|
||||
td, th { text-align:left; padding:.2rem .5rem; border-bottom:1px solid #2a2a2a; }
|
||||
th { color:var(--muted); font-weight:normal; font-size:.75rem; text-transform:uppercase; letter-spacing:.06em; }
|
||||
kbd { background:#2a2a2a; border:1px solid #444; border-radius:3px; padding:0 .3rem; font-size:.75rem; }
|
||||
h2 { font-size:.8rem; letter-spacing:.08em; text-transform:uppercase; color:var(--amber); margin:0 0 .6rem; }
|
||||
.row { display:flex; flex-wrap:wrap; gap:.6rem; align-items:center; }
|
||||
select, input { background:#2a2a2a; color:var(--text); border:1px solid #444; border-radius:4px; padding:.4rem .5rem; font:inherit; }
|
||||
input[type=number] { width:5rem; }
|
||||
label { color:var(--muted); font-size:.8rem; }
|
||||
pre { background:#0d0d0d; border:1px solid #2a2a2a; border-radius:4px; padding:.6rem; max-height:22rem; overflow:auto; font-size:.75rem; white-space:pre-wrap; margin:.6rem 0 0; }
|
||||
.ok { color:var(--green); }
|
||||
.bad { color:var(--red); }
|
||||
button:disabled { opacity:.45; cursor:not-allowed; }
|
||||
button.small { padding:.25rem .5rem; font-size:.75rem; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
@@ -46,6 +56,10 @@ export function reviewPage(): string {
|
||||
<section class="card">
|
||||
<table id="stats"><thead><tr><th>booth</th><th>operator</th><th>reviewed</th><th>agree</th><th>disagree</th><th>unusable</th></tr></thead><tbody></tbody></table>
|
||||
</section>
|
||||
<section class="card" id="training" hidden>
|
||||
<h2>Training</h2>
|
||||
<div id="tr-body"></div>
|
||||
</section>
|
||||
<p class="muted">Keys: <kbd>1</kbd>–<kbd>9</kbd>, <kbd>0</kbd> pick a class in order · <kbd>u</kbd> unusable · <kbd>s</kbd> skip. Skipped items come back after a reload. Your verdict is the training label; the operator's pick is only compared against it.</p>
|
||||
</main>
|
||||
<script>
|
||||
@@ -114,6 +128,106 @@ document.addEventListener('keydown', e => {
|
||||
|
||||
next().catch(e => { document.getElementById('item').innerHTML = '<p class="warn">' + esc(e.message) + '</p>'; });
|
||||
loadStats().catch(() => {});
|
||||
|
||||
// ---- Training: the trainer's job API, proxied by the collector -------------------------
|
||||
// Readiness (labels per class vs the minimum), one job at a time with a live log, the
|
||||
// versions a run produced (written or refused) with Report / Evaluate / Publish. Pinning a
|
||||
// published version into the vision image stays a git commit — that is the deploy control.
|
||||
let trPoll = null;
|
||||
let trShownReport = null;
|
||||
const trDefaults = { mode: 'features', backbone: 'resnet18', minAccuracy: 0.85 };
|
||||
|
||||
function pct(x) { return x == null ? '—' : Math.round(x * 100) + ' %'; }
|
||||
|
||||
async function training() {
|
||||
const box = document.getElementById('training');
|
||||
const el = document.getElementById('tr-body');
|
||||
let s;
|
||||
try { s = await api('/api/training/status'); } catch (e) { box.hidden = false; el.innerHTML = '<p class="warn">' + esc(e.message) + '</p>'; return; }
|
||||
if (!s.configured) { box.hidden = true; return; }
|
||||
box.hidden = false;
|
||||
if (!s.reachable) { el.innerHTML = '<p class="warn">trainer not reachable: ' + esc(s.error || '') + '</p>'; schedule(true); return; }
|
||||
const r = s.readiness, run = r.run || {}, minPer = run.minPerClass || 20;
|
||||
const byClass = (r.labelled && r.labelled.byClass) || {};
|
||||
const classes = Object.keys(byClass);
|
||||
const cur = s.current;
|
||||
const readyLine = r.ready
|
||||
? '<span class="ok">enough labels to train</span> — classes this run: ' + esc((run.classes || []).join(', '))
|
||||
: '<span class="warn">not enough labels yet</span> — a class needs ' + minPer + ' reviewed crops; two classes must clear it';
|
||||
let html = '<p>' + readyLine + ' <span class="muted">(' + (r.labelled ? r.labelled.total : 0) + ' labelled, ' + (r.missingCrops || 0) + ' missing crop files)</span></p>';
|
||||
html += '<table><thead><tr><th>class</th><th>reviewed</th><th>train</th><th>val</th><th></th></tr></thead><tbody>' +
|
||||
(classes.map(c => '<tr><td class="mono">' + esc(c) + '</td><td>' + byClass[c] + '</td><td>' + ((run.train || {})[c] ?? '—') + '</td><td>' + ((run.val || {})[c] ?? '—') + '</td><td class="muted">' + (byClass[c] < minPer ? 'below ' + minPer + ' — dropped' : '') + '</td></tr>').join('') || '<tr><td colspan="5" class="muted">no labels yet — review crops above</td></tr>') +
|
||||
'</tbody></table>';
|
||||
const d = Object.assign({}, trDefaults, r.defaults || {});
|
||||
html += '<div class="row" style="margin-top:.8rem">' +
|
||||
'<label>mode <select id="tr-mode">' + (r.modes || ['features', 'finetune']).map(m => '<option' + (m === d.mode ? ' selected' : '') + '>' + m + '</option>').join('') + '</select></label>' +
|
||||
'<label>backbone <select id="tr-backbone">' + (r.backbones || ['resnet18']).map(b => '<option' + (b === d.backbone ? ' selected' : '') + '>' + b + '</option>').join('') + '</select></label>' +
|
||||
'<label>floor <input id="tr-floor" type="number" min="0" max="1" step="0.01" value="' + d.minAccuracy + '"></label>' +
|
||||
'<button id="tr-train"' + (r.ready && !cur ? '' : ' disabled') + '>Train</button>' +
|
||||
(cur ? '<span class="warn">running: ' + esc(cur.kind) + ' ' + esc(cur.id) + '</span>' : '') +
|
||||
'</div>';
|
||||
const last = cur || (s.jobs && s.jobs[0]);
|
||||
if (last) {
|
||||
const cls = last.status === 'done' ? 'ok' : last.status === 'running' ? 'warn' : 'bad';
|
||||
html += '<p style="margin:.8rem 0 0"><span class="' + cls + '">' + esc(last.status) + '</span> <span class="mono">' + esc(last.kind) + ' ' + esc(last.id) + '</span> <span class="muted">' + esc(last.startedAt || '') + (last.exitCode != null ? ' · exit ' + last.exitCode : '') + '</span> <button class="small" data-job="' + esc(last.id) + '">log</button></p>' +
|
||||
'<pre id="tr-log" hidden></pre>';
|
||||
}
|
||||
const vs = s.versions || [];
|
||||
html += '<h2 style="margin-top:1rem">Versions</h2>';
|
||||
html += vs.length
|
||||
? '<table><thead><tr><th>version</th><th>model</th><th>accuracy</th><th>classes</th><th>mode</th><th></th></tr></thead><tbody>' +
|
||||
vs.map(v => '<tr><td class="mono">' + esc(v.version) + '</td><td>' + (v.written ? '<span class="ok">written</span>' : '<span class="bad">refused</span>') + '</td><td>' + pct(v.accuracy) + (v.floor != null ? ' <span class="muted">/ floor ' + pct(v.floor) + '</span>' : '') + '</td><td class="muted">' + esc((v.classes || []).join(', ')) + '</td><td class="muted">' + esc(v.mode || '') + '</td><td>' +
|
||||
'<button class="small" data-report="' + esc(v.version) + '">report</button> ' +
|
||||
(v.written ? '<button class="small" data-eval="' + esc(v.version) + '"' + (cur ? ' disabled' : '') + '>evaluate</button> <button class="small" data-publish="' + esc(v.version) + '"' + (cur ? ' disabled' : '') + '>publish</button>' : '') +
|
||||
'</td></tr>').join('') + '</tbody></table>'
|
||||
: '<p class="muted">no runs yet</p>';
|
||||
html += '<pre id="tr-report" hidden></pre>';
|
||||
html += '<p class="muted" style="margin:.8rem 0 0">A written model is only a file here. To put it on a booth: publish, then pin the version in <span class="mono">apps/vision/models/bodytype.version</span>, commit, and bump the TAG of the booth.</p>';
|
||||
el.innerHTML = html;
|
||||
|
||||
const trainBtn = document.getElementById('tr-train');
|
||||
if (trainBtn) trainBtn.addEventListener('click', () => startJob({ kind: 'train', mode: document.getElementById('tr-mode').value, backbone: document.getElementById('tr-backbone').value, minAccuracy: Number(document.getElementById('tr-floor').value) }));
|
||||
el.querySelectorAll('button[data-eval]').forEach(b => b.addEventListener('click', () => startJob({ kind: 'evaluate', version: b.dataset.eval })));
|
||||
el.querySelectorAll('button[data-publish]').forEach(b => b.addEventListener('click', () => { if (confirm('Publish ' + b.dataset.publish + ' to the package registry?')) startJob({ kind: 'publish', version: b.dataset.publish }); }));
|
||||
el.querySelectorAll('button[data-job]').forEach(b => b.addEventListener('click', () => showLog(b.dataset.job)));
|
||||
el.querySelectorAll('button[data-report]').forEach(b => b.addEventListener('click', () => showReport(b.dataset.report)));
|
||||
if (cur) showLog(cur.id).catch(() => {});
|
||||
if (trShownReport) showReport(trShownReport).catch(() => {});
|
||||
schedule(!!cur);
|
||||
}
|
||||
|
||||
function schedule(soon) {
|
||||
if (trPoll) clearTimeout(trPoll);
|
||||
trPoll = setTimeout(() => training().catch(() => {}), soon ? 4000 : 60000);
|
||||
}
|
||||
|
||||
async function startJob(body) {
|
||||
try {
|
||||
const r = await fetch('/api/training/jobs', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) });
|
||||
if (!r.ok) { const e = await r.json().catch(() => ({})); alert('trainer: ' + (e.error || ('HTTP ' + r.status))); }
|
||||
} catch (e) { alert(e.message); }
|
||||
training().catch(() => {});
|
||||
}
|
||||
|
||||
async function showLog(id) {
|
||||
const j = await api('/api/training/jobs/' + encodeURIComponent(id));
|
||||
const pre = document.getElementById('tr-log');
|
||||
if (!pre) return;
|
||||
pre.hidden = false;
|
||||
pre.textContent = j.log || '(no output yet)';
|
||||
pre.scrollTop = pre.scrollHeight;
|
||||
}
|
||||
|
||||
async function showReport(v) {
|
||||
const r = await fetch('/api/training/versions/' + encodeURIComponent(v) + '/report');
|
||||
const pre = document.getElementById('tr-report');
|
||||
if (!pre) return;
|
||||
trShownReport = v;
|
||||
pre.hidden = false;
|
||||
pre.textContent = r.ok ? await r.text() : 'no report for ' + v + ' (HTTP ' + r.status + ')';
|
||||
}
|
||||
|
||||
training().catch(() => {});
|
||||
</script>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { buildCollector, type CollectorApp } from "./app.js";
|
||||
|
||||
// The Training section's proxy: reviewer-gated, forwards a fixed set of paths to the
|
||||
// trainer's job API, passes its status codes through, and degrades cleanly when the trainer
|
||||
// is not configured or not reachable. The trainer is faked with a bare node http server.
|
||||
|
||||
const REVIEWER = { user: "julian", pass: "review-pass-123" };
|
||||
const basic = "Basic " + Buffer.from(`${REVIEWER.user}:${REVIEWER.pass}`).toString("base64");
|
||||
|
||||
let dir: string;
|
||||
let fake: Server;
|
||||
let fakeUrl: string;
|
||||
let seen: { method: string; url: string; body: string }[];
|
||||
let app: CollectorApp;
|
||||
|
||||
async function start(trainerUrl: string | null): Promise<void> {
|
||||
app = await buildCollector({ host: "127.0.0.1", port: 0, dataDir: dir, boothTokens: new Map(), reviewer: REVIEWER, trainerUrl }, { dbFile: ":memory:" });
|
||||
await app.ready();
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
dir = await mkdtemp(path.join(tmpdir(), "collector-"));
|
||||
seen = [];
|
||||
fake = createServer((req: IncomingMessage, res: ServerResponse) => {
|
||||
let body = "";
|
||||
req.on("data", (c) => (body += c));
|
||||
req.on("end", () => {
|
||||
seen.push({ method: req.method ?? "", url: req.url ?? "", body });
|
||||
const json = (code: number, obj: unknown) => {
|
||||
res.writeHead(code, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(obj));
|
||||
};
|
||||
if (req.url === "/health") return json(200, { ok: true, busy: false });
|
||||
if (req.url === "/readiness") return json(200, { ready: false, labelled: { total: 3 } });
|
||||
if (req.url === "/versions") return json(200, { versions: [{ version: "v1", written: true }] });
|
||||
if (req.url === "/jobs" && req.method === "GET") return json(200, { jobs: [{ id: "j1" }], current: null });
|
||||
if (req.url === "/jobs" && req.method === "POST") return body.includes('"busy"') ? json(409, { error: "a job is already running" }) : json(202, { id: "j2", status: "running" });
|
||||
if (req.url === "/jobs/j1") return json(200, { id: "j1", status: "done", log: "ok" });
|
||||
if (req.url === "/versions/v1/report") {
|
||||
res.writeHead(200, { "content-type": "text/markdown; charset=utf-8" });
|
||||
return res.end("# Body-type classifier v1\n");
|
||||
}
|
||||
return json(404, { error: "not found" });
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => fake.listen(0, "127.0.0.1", r));
|
||||
const a = fake.address() as { port: number };
|
||||
fakeUrl = `http://127.0.0.1:${a.port}`;
|
||||
});
|
||||
afterEach(async () => {
|
||||
await app?.close();
|
||||
await new Promise<void>((r) => fake.close(() => r()));
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("review page script", () => {
|
||||
it("parses as JavaScript (an apostrophe in a template literal once broke the whole page)", async () => {
|
||||
const { reviewPage } = await import("./review-page.js");
|
||||
const html = reviewPage();
|
||||
const script = html.slice(html.indexOf("<script>") + 8, html.lastIndexOf("</script>"));
|
||||
expect(() => new Function(script)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("training proxy", () => {
|
||||
it("is hidden when no trainer is configured", async () => {
|
||||
await start(null);
|
||||
const s = await app.inject({ method: "GET", url: "/api/training/status", headers: { authorization: basic } });
|
||||
expect(s.json()).toEqual({ configured: false });
|
||||
const j = await app.inject({ method: "POST", url: "/api/training/jobs", headers: { authorization: basic }, payload: { kind: "train" } });
|
||||
expect(j.statusCode).toBe(503);
|
||||
});
|
||||
|
||||
it("aggregates status and forwards jobs and reports behind the reviewer login", async () => {
|
||||
await start(fakeUrl);
|
||||
expect((await app.inject({ method: "GET", url: "/api/training/status" })).statusCode).toBe(401);
|
||||
const s = await app.inject({ method: "GET", url: "/api/training/status", headers: { authorization: basic } });
|
||||
expect(s.statusCode).toBe(200);
|
||||
const body = s.json();
|
||||
expect(body.configured).toBe(true);
|
||||
expect(body.reachable).toBe(true);
|
||||
expect(body.readiness.labelled.total).toBe(3);
|
||||
expect(body.versions[0].version).toBe("v1");
|
||||
expect(body.jobs[0].id).toBe("j1");
|
||||
|
||||
const j = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/training/jobs",
|
||||
headers: { authorization: basic },
|
||||
payload: { kind: "train", mode: "features", minAccuracy: 0.9, secret: "nope", version: "v2" },
|
||||
});
|
||||
expect(j.statusCode).toBe(202);
|
||||
expect(j.json().id).toBe("j2");
|
||||
const posted = seen.find((r) => r.method === "POST")!;
|
||||
expect(JSON.parse(posted.body)).toEqual({ kind: "train", mode: "features", minAccuracy: 0.9, version: "v2" }); // unknown keys dropped
|
||||
|
||||
const busy = await app.inject({ method: "POST", url: "/api/training/jobs", headers: { authorization: basic }, payload: { kind: "evaluate", version: "busy" } });
|
||||
expect(busy.statusCode).toBe(409); // the trainer's answer passes through
|
||||
|
||||
const bad = await app.inject({ method: "POST", url: "/api/training/jobs", headers: { authorization: basic }, payload: { kind: "rm-rf" } });
|
||||
expect(bad.statusCode).toBe(400);
|
||||
|
||||
const one = await app.inject({ method: "GET", url: "/api/training/jobs/j1", headers: { authorization: basic } });
|
||||
expect(one.json().status).toBe("done");
|
||||
expect((await app.inject({ method: "GET", url: "/api/training/jobs/..%2Fx", headers: { authorization: basic } })).statusCode).toBe(400);
|
||||
|
||||
const rep = await app.inject({ method: "GET", url: "/api/training/versions/v1/report", headers: { authorization: basic } });
|
||||
expect(rep.statusCode).toBe(200);
|
||||
expect(rep.headers["content-type"]).toContain("text/markdown");
|
||||
expect(rep.body).toContain("# Body-type classifier v1");
|
||||
});
|
||||
|
||||
it("reports an unreachable trainer without failing the page", async () => {
|
||||
await start("http://127.0.0.1:9"); // nothing listens on the discard port
|
||||
const s = await app.inject({ method: "GET", url: "/api/training/status", headers: { authorization: basic } });
|
||||
expect(s.statusCode).toBe(200);
|
||||
expect(s.json().reachable).toBe(false);
|
||||
const j = await app.inject({ method: "POST", url: "/api/training/jobs", headers: { authorization: basic }, payload: { kind: "train" } });
|
||||
expect(j.statusCode).toBe(502);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "https://schema.tauri.app/config/2",
|
||||
"productName": "Parking System",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0",
|
||||
"identifier": "com.parking.desktop",
|
||||
"build": {
|
||||
"devUrl": "http://localhost:5173",
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
.venv/
|
||||
**/__pycache__/
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
out/
|
||||
.env
|
||||
@@ -0,0 +1,12 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.venv/
|
||||
.mypy_cache/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
|
||||
# Model weights (fetched at deploy / first run, never committed — can be large + license-scoped)
|
||||
out/
|
||||
|
||||
*.onnx
|
||||
@@ -0,0 +1 @@
|
||||
3.12
|
||||
@@ -0,0 +1,49 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# Parking TRAINER image: the phase-B body-type classifier. Build CONTEXT is apps/trainer
|
||||
# (self-contained Python package). Runs on the reviewer's host (art-docker-station) beside
|
||||
# the collector, never on a booth: by default it SERVES the job API the collector's Training
|
||||
# section drives (`serve`); the same image runs the CLI one-off (`train`, `inspect`, …). It
|
||||
# reads the wash collector's volume (collector.sqlite + crops/) and writes versioned model
|
||||
# folders. CPU-only PyTorch — the host has no usable GPU and a few thousand crops train in
|
||||
# minutes/an hour on four Xeon cores.
|
||||
# See wiki/decisions/bodytype-classifier-training.md.
|
||||
|
||||
FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS base
|
||||
WORKDIR /app
|
||||
ENV UV_LINK_MODE=copy \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libgl1 libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY pyproject.toml uv.lock .python-version ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-install-project --no-dev --extra train
|
||||
|
||||
COPY trainer/ ./trainer/
|
||||
COPY README.md ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-dev --extra train
|
||||
|
||||
# Pre-warm the ImageNet backbone weights INTO the image so a run needs no network (the
|
||||
# host has one, but a job that fetches at run time is a job that fails at 2 am). Best-effort:
|
||||
# without network at build time torchvision fetches lazily on the first run.
|
||||
ENV TORCH_HOME=/app/torch-home
|
||||
RUN uv run python -c "import torchvision.models as m; m.resnet18(weights=m.ResNet18_Weights.IMAGENET1K_V1); m.mobilenet_v3_small(weights=m.MobileNet_V3_Small_Weights.IMAGENET1K_V1)" \
|
||||
|| echo "[build] backbone weights not pre-warmed (no network) — fetched on first run"
|
||||
|
||||
RUN useradd --system --create-home --uid 999 trainer \
|
||||
&& mkdir -p /data /out && chown -R trainer:trainer /app /out
|
||||
USER trainer
|
||||
|
||||
ENV TRAINER_DATA_DIR=/data \
|
||||
TRAINER_OUT_DIR=/out \
|
||||
TRAINER_PORT=8091
|
||||
VOLUME ["/out"]
|
||||
EXPOSE 8091
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8091/health').status==200 else 1)" || exit 1
|
||||
ENTRYPOINT ["uv", "run", "--no-sync", "parking-trainer"]
|
||||
CMD ["serve"]
|
||||
@@ -0,0 +1,36 @@
|
||||
# parking-trainer
|
||||
|
||||
The phase-B **body-type classifier** job. Reads the wash collector's volume
|
||||
(`collector.sqlite` + `crops/`), trains a classifier on the reviewer's labels, and writes a
|
||||
versioned model folder the vision image bakes in — or refuses when validation is below the
|
||||
floor. Design and decisions: `wiki/decisions/bodytype-classifier-training.md`.
|
||||
|
||||
```
|
||||
parking-trainer inspect --data /data # what a run would train on
|
||||
parking-trainer train --data /data --out /out # features mode (minutes)
|
||||
parking-trainer train --mode finetune --epochs 12 ... # full fine-tune (about an hour on 4 cores)
|
||||
parking-trainer evaluate --model /out/<version>/bodytype.onnx --data /data
|
||||
parking-trainer publish /out/<version> --url https://git.infra.msai.al/api/packages/mca/generic/parking-bodytype
|
||||
```
|
||||
|
||||
Exit codes: `0` model written · `2` not enough labels · `3` below the floor (report written,
|
||||
no model) · `1` other.
|
||||
|
||||
A passing run writes `<out>/<version>/`:
|
||||
|
||||
| file | what |
|
||||
| --- | --- |
|
||||
| `bodytype.onnx` | the classifier; input `image` = RGB float32 0–255 `[N,3,S,S]`, output `logits` `[N,K]`; normalisation is inside the graph |
|
||||
| `bodytype.json` | sidecar: version, class list (in vocabulary order), input size, crop margin, backbone, mode, label counts, validation metrics |
|
||||
| `report.md` | the human report: accuracy, per-class recall/precision, confusion matrix, dropped classes, loss weights |
|
||||
| `metrics.json` | the same numbers, machine-readable |
|
||||
|
||||
On the reviewer's host the image runs `serve` as the `trainer` service of the
|
||||
`wash-collector` stack: a job API (`/health`, `/readiness`, `/versions`, `/jobs`) on the compose
|
||||
network that the collector's **Training section** (`/review`) drives — readiness, Train /
|
||||
Evaluate / Publish, reports and logs. Jobs run as subprocesses of the CLI, one at a time; state
|
||||
and logs persist under `/out/jobs/`. The CLI stays for debugging:
|
||||
`docker compose -f docker-compose.collector.yml exec trainer parking-trainer inspect`.
|
||||
|
||||
Local dev: `uv sync --extra train` (CPU torch, ~200 MB), `uv run pytest -q`. The test suite
|
||||
runs without the extra (torch tests skip), matching CI.
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "@parking/trainer",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"//": "Thin shim so this Python job is a node in the Turbo task graph (NOT a JS package — deps are managed by uv/pyproject.toml). It is a one-off job image, never a booth service: see wiki/decisions/bodytype-classifier-training.md.",
|
||||
"scripts": {
|
||||
"lint": "uv run ruff check .",
|
||||
"format": "uv run ruff format .",
|
||||
"typecheck": "uv run mypy trainer",
|
||||
"test": "uv run pytest -q",
|
||||
"build": "echo 'no build step (Python job; see Dockerfile)'"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
[project]
|
||||
name = "parking-trainer"
|
||||
version = "0.0.0"
|
||||
description = "Phase-B body-type classifier trainer: reviewer labels + crops off the wash collector's volume → an ONNX classifier the vision image bakes in."
|
||||
requires-python = ">=3.10,<4.0"
|
||||
# Core deps are LIGHT on purpose (same rule as the vision service): `inspect`, `evaluate`
|
||||
# and the data/report code run with only these, so `uv sync` and the test suite work
|
||||
# in CI without the PyTorch stack. Training itself needs the `train` extra.
|
||||
# See wiki/decisions/bodytype-classifier-training.md.
|
||||
dependencies = [
|
||||
"numpy>=1.26",
|
||||
# OpenCV does the decode + resize on BOTH sides (trainer and vision service): same
|
||||
# library, same interpolation, same pixels — the preprocessing contract (preprocess.py).
|
||||
"opencv-python-headless>=4.10",
|
||||
"onnxruntime>=1.19",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
parking-trainer = "trainer.cli:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
# The training stack. CPU-only PyTorch (the reviewer's host has no usable GPU — the
|
||||
# decision is recorded in the wiki page above): resolved from PyTorch's CPU wheel index,
|
||||
# ~200 MB instead of the ~5 GB CUDA build. Install with: uv sync --extra train
|
||||
# torch / torchvision are BSD-3; the ImageNet backbone weights ship under the same
|
||||
# licence (the licence rule applies to weights as much as code).
|
||||
train = [
|
||||
"torch>=2.4",
|
||||
"torchvision>=0.19",
|
||||
"onnx>=1.16",
|
||||
"onnxscript>=0.3", # the torch.export-based ONNX exporter (MIT)
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"ruff>=0.8",
|
||||
"pytest>=8.3",
|
||||
"mypy>=1.13",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
# Pick the CPU wheels for torch/torchvision from PyTorch's own index; everything else
|
||||
# from PyPI. `explicit = true` keeps the index from shadowing PyPI for other packages.
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cpu"
|
||||
url = "https://download.pytorch.org/whl/cpu"
|
||||
explicit = true
|
||||
|
||||
[tool.uv.sources]
|
||||
torch = [{ index = "pytorch-cpu" }]
|
||||
torchvision = [{ index = "pytorch-cpu" }]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 110
|
||||
target-version = "py310"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "B", "UP"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.12"
|
||||
strict = true
|
||||
ignore_missing_imports = true
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["trainer"]
|
||||
@@ -0,0 +1,102 @@
|
||||
"""A synthetic collector volume: the collector's `items` table (same DDL as apps/collector
|
||||
src/db.ts) + JPEG crops. Classes are told apart by COLOUR so even a random-init backbone's
|
||||
features separate them — the tests check the plumbing (split, floor, export, sidecar),
|
||||
not accuracy on real cars."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
DDL = """
|
||||
CREATE TABLE IF NOT EXISTS items (
|
||||
id TEXT PRIMARY KEY, booth TEXT NOT NULL, kind TEXT NOT NULL DEFAULT 'wash',
|
||||
order_ref TEXT NOT NULL, at TEXT NOT NULL, operator_ref TEXT NOT NULL DEFAULT '',
|
||||
operator_category_id TEXT NOT NULL DEFAULT '', operator_category_name TEXT NOT NULL DEFAULT '',
|
||||
operator_classes TEXT NOT NULL DEFAULT '[]', service TEXT NOT NULL, vision_class TEXT NOT NULL,
|
||||
vision_confidence REAL NOT NULL, vision_category_id TEXT, downgraded INTEGER NOT NULL DEFAULT 0,
|
||||
image_width INTEGER NOT NULL, image_height INTEGER NOT NULL, plate_blurred INTEGER NOT NULL,
|
||||
image_path TEXT NOT NULL, received_at TEXT NOT NULL, review_label TEXT, reviewed_at TEXT, reviewer TEXT
|
||||
);
|
||||
"""
|
||||
|
||||
COLOURS = {"sedan": (200, 40, 40), "suv": (40, 200, 40), "van": (40, 40, 200), "truck": (200, 200, 40)}
|
||||
|
||||
|
||||
def write_jpeg(path: Path, colour: tuple[int, int, int], rng: np.random.Generator) -> None:
|
||||
import cv2
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
h, w = int(rng.integers(120, 200)), int(rng.integers(160, 260))
|
||||
img = np.empty((h, w, 3), np.uint8)
|
||||
img[:] = colour[::-1] # BGR
|
||||
noise = rng.integers(-20, 20, size=img.shape, dtype=np.int16)
|
||||
img = np.clip(img.astype(np.int16) + noise, 0, 255).astype(np.uint8)
|
||||
cv2.imwrite(str(path), img, [cv2.IMWRITE_JPEG_QUALITY, 85])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collector_dir(tmp_path: Path) -> Path:
|
||||
"""40 labelled crops per class for sedan/suv/van, 5 for truck (below the minimum), a few
|
||||
unusable, a few pending, one labelled row whose file is missing."""
|
||||
rng = np.random.default_rng(1)
|
||||
con = sqlite3.connect(tmp_path / "collector.sqlite")
|
||||
con.executescript(DDL)
|
||||
t0 = datetime(2026, 9, 1, tzinfo=timezone.utc)
|
||||
n = 0
|
||||
|
||||
def add(label: str | None, reviewed: bool, kind: str = "wash", missing: bool = False) -> None:
|
||||
nonlocal n
|
||||
n += 1
|
||||
item = f"item-{n:04d}"
|
||||
rel = f"crops/booth-2/{item}.jpg"
|
||||
colour = COLOURS.get(label or "sedan", (128, 128, 128))
|
||||
if not missing:
|
||||
write_jpeg(tmp_path / rel, colour, rng)
|
||||
at = (t0 + timedelta(minutes=10 * n)).isoformat().replace("+00:00", "Z")
|
||||
reviewed_at = (
|
||||
(t0 + timedelta(days=1, minutes=n)).isoformat().replace("+00:00", "Z") if reviewed else None
|
||||
)
|
||||
con.execute(
|
||||
"INSERT INTO items (id, booth, kind, order_ref, at, service, vision_class, vision_confidence, "
|
||||
"image_width, image_height, plate_blurred, image_path, received_at, review_label, "
|
||||
"reviewed_at, reviewer) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
item,
|
||||
"booth-2",
|
||||
kind,
|
||||
"o",
|
||||
at,
|
||||
"wash",
|
||||
"car" if label != "truck" else "truck",
|
||||
0.9,
|
||||
200,
|
||||
150,
|
||||
1,
|
||||
rel,
|
||||
at,
|
||||
label if reviewed else None,
|
||||
reviewed_at,
|
||||
"reviewer" if reviewed else None,
|
||||
),
|
||||
)
|
||||
|
||||
# Interleaved in time so every class exists on both sides of the time split.
|
||||
for i in range(40):
|
||||
for label in ("sedan", "suv", "van"):
|
||||
add(label, True)
|
||||
if i % 8 == 0:
|
||||
add("truck", True)
|
||||
add("unusable", True)
|
||||
add("unusable", True)
|
||||
add("sedan", True, missing=True)
|
||||
for _ in range(6):
|
||||
add(None, False, kind="entry")
|
||||
con.commit()
|
||||
con.close()
|
||||
return tmp_path
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Data rules, torch-free: labels, the time split, thin classes, weights, the report."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from trainer.cli import main
|
||||
from trainer.data import (
|
||||
class_weights,
|
||||
load_labelled,
|
||||
load_reviewed_since,
|
||||
load_unlabelled,
|
||||
make_split,
|
||||
summarise,
|
||||
)
|
||||
from trainer.preprocess import CROP_MARGIN, Sidecar, load_input
|
||||
from trainer.report import compute_metrics, render_report
|
||||
|
||||
|
||||
def test_loads_only_reviewed_usable_rows_with_a_crop_on_disk(collector_dir: Path) -> None:
|
||||
samples, missing = load_labelled(collector_dir)
|
||||
assert missing == 1 # the labelled row whose file is gone
|
||||
assert len(samples) == 125 # 3×40 + 5 trucks; unusable and pending excluded
|
||||
assert all(s.path.is_file() for s in samples)
|
||||
assert {s.label for s in samples} == {"sedan", "suv", "van", "truck"}
|
||||
assert summarise(samples)["byClass"] == {"sedan": 40, "suv": 40, "van": 40, "truck": 5}
|
||||
assert len(load_unlabelled(collector_dir)) == 6
|
||||
assert len(load_reviewed_since(collector_dir, "2026-09-02T00:00:00Z")) == 125
|
||||
assert load_reviewed_since(collector_dir, "2030-01-01T00:00:00Z") == []
|
||||
|
||||
|
||||
def test_split_is_by_time_and_drops_thin_classes(collector_dir: Path) -> None:
|
||||
samples, _ = load_labelled(collector_dir)
|
||||
split = make_split(samples, val_fraction=0.2, min_per_class=20)
|
||||
assert split.classes == ("sedan", "suv", "van") # canonical order, truck dropped
|
||||
assert split.dropped == {"truck": 5}
|
||||
assert len(split.train) + len(split.val) == 120
|
||||
assert len(split.val) == 24
|
||||
assert max(s.at for s in split.train) < min(s.at for s in split.val) # newest = validation
|
||||
assert all(v > 0 for v in split.counts("val").values())
|
||||
|
||||
|
||||
def test_class_weights_lean_against_imbalance_but_gently(collector_dir: Path) -> None:
|
||||
samples, _ = load_labelled(collector_dir)
|
||||
vans = [s for s in samples if s.label == "van"]
|
||||
keep = set(vans[::10]) # 4 of 40 vans survive
|
||||
split = make_split([s for s in samples if s.label != "van" or s in keep], 0.2, 3)
|
||||
w = dict(zip(split.classes, class_weights(split), strict=True))
|
||||
assert w["van"] > w["sedan"] > 0 # the rare class weighs more
|
||||
assert w["van"] / w["sedan"] < 4 # but not the full inverse ratio (damped)
|
||||
assert abs(sum(w.values()) / len(w) - 1.0) < 1e-9
|
||||
|
||||
|
||||
def test_metrics_and_report() -> None:
|
||||
classes = ("sedan", "suv")
|
||||
m = compute_metrics(classes, [0, 0, 1, 1], [0, 1, 1, 1], camera=["car"] * 4)
|
||||
assert m.accuracy == 0.75
|
||||
assert m.per_class["sedan"].recall == 0.5 and m.per_class["suv"].precision == 2 / 3
|
||||
assert m.confusion == [[1, 1], [0, 2]]
|
||||
assert m.camera_agreement == 0.0
|
||||
text = render_report(
|
||||
version="v1",
|
||||
trained_at="t",
|
||||
mode="features",
|
||||
backbone="resnet18",
|
||||
epochs=3,
|
||||
classes=classes,
|
||||
train_counts={"sedan": 10, "suv": 8},
|
||||
val_counts={"sedan": 2, "suv": 2},
|
||||
dropped={"truck": 2},
|
||||
missing_files=1,
|
||||
weights=[0.9, 1.1],
|
||||
metrics=m,
|
||||
min_accuracy=0.85,
|
||||
written=False,
|
||||
)
|
||||
assert "MODEL NOT WRITTEN" in text and "| **sedan** | 1 | 1 |" in text and "truck (2)" in text
|
||||
|
||||
|
||||
def test_preprocess_contract(tmp_path: Path, collector_dir: Path) -> None:
|
||||
samples, _ = load_labelled(collector_dir)
|
||||
x = load_input(samples[0].path, 32)
|
||||
assert x.shape == (3, 32, 32) and x.dtype.name == "float32" and 0 <= x.min() and x.max() <= 255
|
||||
assert x[0].mean() > x[2].mean() # a sedan crop is red: RGB order, not BGR
|
||||
assert load_input(tmp_path / "nope.jpg", 32) is None
|
||||
side = Sidecar(version="v1", classes=["sedan", "suv"])
|
||||
side.write(tmp_path / "s.json")
|
||||
back = Sidecar.read(tmp_path / "s.json")
|
||||
assert back == side and back.crop_margin == CROP_MARGIN == 0.08 and back.normalization == "in-graph"
|
||||
|
||||
|
||||
def test_inspect_prints_the_run_shape(collector_dir: Path, capsys) -> None: # type: ignore[no-untyped-def]
|
||||
assert main(["inspect", "--data", str(collector_dir)]) == 0
|
||||
out = json.loads(capsys.readouterr().out)
|
||||
assert out["ready"] is True and out["run"]["classes"] == ["sedan", "suv", "van"]
|
||||
assert out["run"]["dropped"] == {"truck": 5} and out["missingCrops"] == 1
|
||||
assert main(["inspect", "--data", str(collector_dir), "--min-per-class", "100"]) == 2
|
||||
@@ -0,0 +1,147 @@
|
||||
"""The job API: readiness, one job at a time, subprocess jobs with persisted logs, versions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from trainer.server import Handler, Jobs, readiness, versions, wait_idle
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api(collector_dir: Path, tmp_path: Path): # type: ignore[no-untyped-def]
|
||||
out = tmp_path / "out"
|
||||
Handler.jobs = Jobs(collector_dir, out, "https://example.invalid/pkg", "tok")
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
t = threading.Thread(target=httpd.serve_forever, daemon=True)
|
||||
t.start()
|
||||
base = f"http://127.0.0.1:{httpd.server_address[1]}"
|
||||
|
||||
def call(method: str, path: str, body: dict | None = None): # type: ignore[no-untyped-def]
|
||||
req = urllib.request.Request(base + path, method=method)
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
req.add_header("Content-Type", "application/json")
|
||||
try:
|
||||
with urllib.request.urlopen(req, data=data, timeout=10) as r:
|
||||
raw = r.read()
|
||||
return r.status, (
|
||||
json.loads(raw) if r.headers.get_content_type() == "application/json" else raw.decode()
|
||||
)
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, json.loads(e.read() or b"{}")
|
||||
|
||||
yield call, out
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
|
||||
|
||||
def test_readiness_and_empty_versions(api) -> None: # type: ignore[no-untyped-def]
|
||||
call, _ = api
|
||||
code, r = call("GET", "/readiness")
|
||||
assert code == 200 and r["ready"] is True and r["run"]["classes"] == ["sedan", "suv", "van"]
|
||||
assert r["defaults"]["minAccuracy"] == 0.85 and "finetune" in r["modes"]
|
||||
assert call("GET", "/versions") == (200, {"versions": []})
|
||||
assert call("GET", "/health")[1]["busy"] is False
|
||||
assert readiness(Path("/nonexistent"))["ready"] is False
|
||||
|
||||
|
||||
def test_evaluate_job_runs_as_a_subprocess_and_is_recorded(api) -> None: # type: ignore[no-untyped-def]
|
||||
call, out = api
|
||||
code, job = call("POST", "/jobs", {"kind": "evaluate", "version": "nope"})
|
||||
assert code == 202 and job["status"] == "running" and job["kind"] == "evaluate"
|
||||
wait_idle(Handler.jobs)
|
||||
code, j = call("GET", f"/jobs/{job['id']}")
|
||||
assert code == 200 and j["status"] == "failed" and j["exitCode"] == 1
|
||||
assert "evaluate --data" in j["log"] and "nope" in j["log"]
|
||||
assert (out / "jobs" / f"{job['id']}.json").is_file() and (out / "jobs" / f"{job['id']}.log").is_file()
|
||||
code, lst = call("GET", "/jobs")
|
||||
assert code == 200 and lst["jobs"][0]["id"] == job["id"] and lst["current"] is None
|
||||
|
||||
|
||||
def test_bad_requests(api) -> None: # type: ignore[no-untyped-def]
|
||||
call, _ = api
|
||||
assert call("POST", "/jobs", {"kind": "nuke"})[0] == 400
|
||||
assert call("POST", "/jobs", {"kind": "train", "mode": "magic"})[0] == 400
|
||||
assert call("POST", "/jobs", {"kind": "evaluate", "version": "../etc"})[0] == 400
|
||||
assert call("POST", "/jobs", {"kind": "publish", "version": "v1", "url": "ftp://x"})[0] == 400
|
||||
assert call("GET", "/versions/../x/report")[0] == 400
|
||||
assert call("GET", "/versions/v9/report")[0] == 404
|
||||
assert call("GET", "/jobs/nope")[0] == 404
|
||||
assert call("GET", "/nothing")[0] == 404
|
||||
|
||||
|
||||
def test_train_job_then_versions_and_report(api) -> None: # type: ignore[no-untyped-def]
|
||||
pytest.importorskip("torch")
|
||||
call, out = api
|
||||
body = {"kind": "train", "mode": "features", "minAccuracy": 0.0, "epochs": 100, "version": "vapi"}
|
||||
# The test-only flags are not offered by the API; inject them via the CLI args the runner builds.
|
||||
orig = Jobs._argv
|
||||
|
||||
def patched(self, kind, a): # type: ignore[no-untyped-def]
|
||||
argv = orig(self, kind, a)
|
||||
return argv + ["--no-pretrained", "--input-size", "64", "--no-cache"] if kind == "train" else argv
|
||||
|
||||
Jobs._argv = patched # type: ignore[method-assign]
|
||||
try:
|
||||
code, job = call("POST", "/jobs", body)
|
||||
assert code == 202
|
||||
assert call("POST", "/jobs", {"kind": "evaluate", "version": "vapi"})[0] == 409 # one at a time
|
||||
wait_idle(Handler.jobs, 120)
|
||||
finally:
|
||||
Jobs._argv = orig # type: ignore[method-assign]
|
||||
code, j = call("GET", f"/jobs/{job['id']}")
|
||||
assert j["status"] == "done" and "MODEL WRITTEN" in j["log"]
|
||||
code, v = call("GET", "/versions")
|
||||
assert code == 200 and v["versions"][0]["version"] == "vapi" and v["versions"][0]["written"] is True
|
||||
assert v["versions"][0]["classes"] == ["sedan", "suv", "van"] and v["versions"][0]["accuracy"] >= 0.9
|
||||
code, report = call("GET", "/versions/vapi/report")
|
||||
assert code == 200 and report.startswith("# Body-type classifier vapi")
|
||||
assert versions(out)[0]["floor"] == 0.0
|
||||
# evaluate on the written model now succeeds
|
||||
code, job2 = call("POST", "/jobs", {"kind": "evaluate", "version": "vapi"})
|
||||
wait_idle(Handler.jobs)
|
||||
assert call("GET", f"/jobs/{job2['id']}")[1]["status"] == "done"
|
||||
|
||||
|
||||
def test_a_handler_error_is_a_500_json_reply_not_a_dropped_connection(tmp_path: Path) -> None:
|
||||
"""A collector DB from before the `kind` column (art-docker-station, 2026-09-16): readiness
|
||||
raised sqlite3.OperationalError, the stdlib server printed the traceback and closed the
|
||||
socket, and the collector could only say "trainer not reachable: fetch failed". The
|
||||
handler must answer 500 JSON naming the error instead."""
|
||||
old = tmp_path / "old"
|
||||
old.mkdir()
|
||||
con = sqlite3.connect(old / "collector.sqlite")
|
||||
con.executescript(
|
||||
"CREATE TABLE items (id TEXT PRIMARY KEY, booth TEXT NOT NULL, order_ref TEXT NOT NULL,"
|
||||
" at TEXT NOT NULL, service TEXT NOT NULL, vision_class TEXT NOT NULL,"
|
||||
" vision_confidence REAL NOT NULL, image_width INTEGER NOT NULL, image_height INTEGER NOT NULL,"
|
||||
" plate_blurred INTEGER NOT NULL, image_path TEXT NOT NULL, received_at TEXT NOT NULL,"
|
||||
" review_label TEXT, reviewed_at TEXT, reviewer TEXT)"
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
Handler.jobs = Jobs(old, tmp_path / "out", "https://example.invalid/pkg", "tok")
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
try:
|
||||
req = urllib.request.Request(f"http://127.0.0.1:{httpd.server_address[1]}/readiness")
|
||||
with pytest.raises(urllib.error.HTTPError) as ei:
|
||||
urllib.request.urlopen(req, timeout=10)
|
||||
assert ei.value.code == 500
|
||||
body = json.loads(ei.value.read())
|
||||
assert "no such column: kind" in body["error"]
|
||||
# /health does not touch the DB and still answers.
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{httpd.server_address[1]}/health", timeout=10) as r:
|
||||
assert r.status == 200
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
@@ -0,0 +1,155 @@
|
||||
"""The training job end to end on the synthetic volume — needs the `train` extra (torch);
|
||||
skipped where it is not installed (CI syncs without it, like the vision service)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
from trainer.cli import main # noqa: E402
|
||||
from trainer.infer import OnnxClassifier # noqa: E402
|
||||
from trainer.preprocess import Sidecar # noqa: E402
|
||||
|
||||
COMMON = ["--no-pretrained", "--input-size", "64", "--no-cache", "--seed", "3"]
|
||||
|
||||
|
||||
def test_features_run_writes_model_sidecar_report_and_evaluates(
|
||||
collector_dir: Path, tmp_path: Path, capsys
|
||||
) -> None: # type: ignore[no-untyped-def]
|
||||
out = tmp_path / "out"
|
||||
rc = main(
|
||||
[
|
||||
"train",
|
||||
"--data",
|
||||
str(collector_dir),
|
||||
"--out",
|
||||
str(out),
|
||||
"--version",
|
||||
"vtest",
|
||||
"--mode",
|
||||
"features",
|
||||
"--epochs",
|
||||
"150",
|
||||
"--min-accuracy",
|
||||
"0.0",
|
||||
*COMMON,
|
||||
]
|
||||
)
|
||||
assert rc == 0
|
||||
d = out / "vtest"
|
||||
assert {p.name for p in d.iterdir()} == {"bodytype.onnx", "bodytype.json", "report.md", "metrics.json"}
|
||||
side = Sidecar.read(d / "bodytype.json")
|
||||
assert side.classes == ["sedan", "suv", "van"] and side.input_size == 64 and side.mode == "features"
|
||||
assert side.labels == {"train": 96, "val": 24} and side.metrics["floor"] == 0.0
|
||||
metrics = json.loads((d / "metrics.json").read_text())
|
||||
assert metrics["n"] == 24 and metrics["onnx_agreement"] == 1.0
|
||||
# Colour-coded classes: even a random backbone's pooled features separate them.
|
||||
assert metrics["accuracy"] >= 0.9
|
||||
report = (d / "report.md").read_text()
|
||||
assert (
|
||||
"MODEL WRITTEN" in report
|
||||
and "truck (5)" in report
|
||||
and "crop is missing on disk (skipped): 1" in report
|
||||
)
|
||||
|
||||
# The exported graph takes raw 0–255 RGB and answers by itself.
|
||||
clf = OnnxClassifier(d / "bodytype.onnx")
|
||||
probs, kept = clf.predict_files(
|
||||
[s for s in sorted((collector_dir / "crops" / "booth-2").glob("*.jpg"))][:6]
|
||||
)
|
||||
assert probs.shape == (6, 3) and kept == [0, 1, 2, 3, 4, 5]
|
||||
assert np.allclose(probs.sum(axis=1), 1.0, atol=1e-4)
|
||||
|
||||
# evaluate: labels reviewed after training (none — the fixture's reviews predate it) and the
|
||||
# unlabelled pile (6 entry samples).
|
||||
capsys.readouterr()
|
||||
assert main(["evaluate", "--data", str(collector_dir), "--model", str(d / "bodytype.onnx")]) == 0
|
||||
res = json.loads(capsys.readouterr().out)
|
||||
assert res["model"] == "vtest" and res["reviewedSince"] is None
|
||||
assert res["unlabelled"]["n"] == 6 and sum(res["unlabelled"]["predicted"].values()) == 6
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"evaluate",
|
||||
"--data",
|
||||
str(collector_dir),
|
||||
"--model",
|
||||
str(d / "bodytype.onnx"),
|
||||
"--since",
|
||||
"2026-09-01T00:00:00Z",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
res2 = json.loads(capsys.readouterr().out)
|
||||
assert res2["reviewedSince"]["n"] == 125 - 5 # trucks are not a class the model knows
|
||||
|
||||
|
||||
def test_below_the_floor_writes_the_report_but_no_model(collector_dir: Path, tmp_path: Path) -> None:
|
||||
out = tmp_path / "out"
|
||||
rc = main(
|
||||
[
|
||||
"train",
|
||||
"--data",
|
||||
str(collector_dir),
|
||||
"--out",
|
||||
str(out),
|
||||
"--version",
|
||||
"vlow",
|
||||
"--mode",
|
||||
"features",
|
||||
"--epochs",
|
||||
"5",
|
||||
"--min-accuracy",
|
||||
"1.01",
|
||||
*COMMON,
|
||||
]
|
||||
)
|
||||
assert rc == 3
|
||||
d = out / "vlow"
|
||||
assert {p.name for p in d.iterdir()} == {"report.md", "metrics.json"}
|
||||
assert "MODEL NOT WRITTEN" in (d / "report.md").read_text()
|
||||
|
||||
|
||||
def test_not_enough_labels_is_exit_2(collector_dir: Path, tmp_path: Path) -> None:
|
||||
out = tmp_path / "out"
|
||||
rc = main(["train", "--data", str(collector_dir), "--out", str(out), "--min-per-class", "100", *COMMON])
|
||||
assert rc == 2
|
||||
assert not out.exists()
|
||||
|
||||
|
||||
def test_finetune_runs_and_uses_the_feature_cache(collector_dir: Path, tmp_path: Path) -> None:
|
||||
out = tmp_path / "out"
|
||||
args = [
|
||||
"train",
|
||||
"--data",
|
||||
str(collector_dir),
|
||||
"--out",
|
||||
str(out),
|
||||
"--mode",
|
||||
"finetune",
|
||||
"--backbone",
|
||||
"mobilenet_v3_small",
|
||||
"--epochs",
|
||||
"1",
|
||||
"--batch",
|
||||
"16",
|
||||
"--min-accuracy",
|
||||
"0.0",
|
||||
"--no-pretrained",
|
||||
"--input-size",
|
||||
"64",
|
||||
"--seed",
|
||||
"3",
|
||||
]
|
||||
assert main([*args, "--version", "vft"]) == 0
|
||||
cache = out / "cache" / "features-mobilenet_v3_small-64.npz"
|
||||
assert cache.exists()
|
||||
z = np.load(cache)
|
||||
assert len(z["ids"]) == 96 and z["feats"].shape == (96, 576)
|
||||
assert Sidecar.read(out / "vft" / "bodytype.json").mode == "finetune"
|
||||
@@ -0,0 +1,7 @@
|
||||
"""parking-trainer — the phase-B body-type classifier job.
|
||||
|
||||
Reads the wash collector's SQLite + crops straight off its volume, splits by TIME, trains a
|
||||
small classifier on a pretrained backbone, and writes the ONNX model + sidecar + report —
|
||||
or refuses to write the model when validation is below the owner's floor.
|
||||
See wiki/decisions/bodytype-classifier-training.md.
|
||||
"""
|
||||
@@ -0,0 +1,414 @@
|
||||
"""parking-trainer — inspect / train / evaluate / publish.
|
||||
|
||||
parking-trainer inspect --data /data
|
||||
parking-trainer train --data /data --out /out [--mode features|finetune] [--min-accuracy 0.85]
|
||||
parking-trainer evaluate --model /out/<version>/bodytype.onnx --data /data
|
||||
parking-trainer publish /out/<version> --url https://<gitea>/api/packages/<owner>/generic/parking-bodytype
|
||||
|
||||
Exit codes: 0 ok · 2 not enough labels · 3 trained but below the floor (report written, model
|
||||
NOT written) · 1 anything else.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from .data import (
|
||||
VEHICLE_CLASSES,
|
||||
class_weights,
|
||||
load_labelled,
|
||||
load_reviewed_since,
|
||||
load_unlabelled,
|
||||
make_split,
|
||||
suggested_epochs,
|
||||
summarise,
|
||||
)
|
||||
from .preprocess import CROP_MARGIN, Sidecar
|
||||
from .report import compute_metrics, render_report
|
||||
|
||||
MODEL_FILE = "bodytype.onnx"
|
||||
SIDECAR_FILE = "bodytype.json"
|
||||
REPORT_FILE = "report.md"
|
||||
METRICS_FILE = "metrics.json"
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
print(f"[trainer] {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# inspect
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def cmd_inspect(a: argparse.Namespace) -> int:
|
||||
samples, missing = load_labelled(a.data)
|
||||
split = make_split(samples, a.val_fraction, a.min_per_class)
|
||||
out = {
|
||||
"labelled": summarise(samples),
|
||||
"missingCrops": missing,
|
||||
"run": {
|
||||
"classes": list(split.classes),
|
||||
"train": split.counts("train"),
|
||||
"val": split.counts("val"),
|
||||
"dropped": split.dropped,
|
||||
"minPerClass": a.min_per_class,
|
||||
"valFraction": a.val_fraction,
|
||||
},
|
||||
"ready": len(split.classes) >= 2,
|
||||
}
|
||||
print(json.dumps(out, indent=2))
|
||||
return 0 if out["ready"] else 2
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# train
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def cmd_train(a: argparse.Namespace) -> int:
|
||||
try:
|
||||
from . import model as M
|
||||
except ImportError as exc: # torch missing
|
||||
_log(f"the training stack is not installed ({exc}); install with: uv sync --extra train")
|
||||
return 1
|
||||
import numpy as np
|
||||
|
||||
samples, missing = load_labelled(a.data)
|
||||
split = make_split(samples, a.val_fraction, a.min_per_class)
|
||||
if len(split.classes) < 2:
|
||||
_log(
|
||||
f"not enough labels: {len(samples)} usable, classes with >= {a.min_per_class}: "
|
||||
f"{list(split.classes)} (dropped {split.dropped}); nothing to train"
|
||||
)
|
||||
return 2
|
||||
version = a.version or datetime.now(timezone.utc).strftime("v%Y%m%d-%H%M")
|
||||
out_dir = a.out / version
|
||||
epochs = a.epochs or suggested_epochs(len(split.train), a.mode)
|
||||
weights = class_weights(split)
|
||||
idx = split.class_index
|
||||
_log(
|
||||
f"{version}: {a.mode} on {a.backbone}, classes {list(split.classes)}, "
|
||||
f"{len(split.train)} train / {len(split.val)} val, {epochs} epochs"
|
||||
)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
x_train, train = M.load_images(split.train, a.input_size)
|
||||
x_val, val = M.load_images(split.val, a.input_size)
|
||||
y_train = [idx[s.label] for s in train]
|
||||
y_val = [idx[s.label] for s in val]
|
||||
_log(f"decoded {len(train)} + {len(val)} crops in {time.perf_counter() - t0:.1f}s")
|
||||
if len(val) == 0 or len(set(y_train)) < 2:
|
||||
_log("not enough decodable crops on both sides of the split")
|
||||
return 2
|
||||
|
||||
backbone = M.build_backbone(a.backbone, pretrained=not a.no_pretrained)
|
||||
cache = (
|
||||
None if a.no_cache else M.FeatureCache(a.out / "cache" / f"features-{a.backbone}-{a.input_size}.npz")
|
||||
)
|
||||
t0 = time.perf_counter()
|
||||
f_train = M.features_for(backbone, train, x_train, cache)
|
||||
_log(
|
||||
f"features for {len(train)} train crops in {time.perf_counter() - t0:.1f}s "
|
||||
f"(cache: {cache.path if cache else 'off'})"
|
||||
)
|
||||
head_epochs = epochs if a.mode == "features" else max(30, epochs * 5)
|
||||
head = M.train_head(f_train, y_train, len(split.classes), weights, head_epochs, seed=a.seed)
|
||||
net = M.Classifier.make(backbone, head)
|
||||
|
||||
if a.mode == "finetune":
|
||||
t0 = time.perf_counter()
|
||||
net = M.finetune(
|
||||
net, x_train, y_train, weights, epochs, batch=a.batch, lr=a.lr, seed=a.seed, log=_log
|
||||
)
|
||||
_log(f"fine-tuned in {(time.perf_counter() - t0) / 60:.1f} min")
|
||||
|
||||
logits = M.predict_logits(net, x_val)
|
||||
y_pred = logits.argmax(axis=1).tolist()
|
||||
metrics = compute_metrics(split.classes, y_val, y_pred, camera=[s.vision_class for s in val])
|
||||
|
||||
# Export and check the graph gives the same answers as the torch model.
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
tmp_model = out_dir / (MODEL_FILE + ".tmp")
|
||||
M.export_onnx(net, a.input_size, tmp_model)
|
||||
from .infer import OnnxClassifier
|
||||
|
||||
sidecar = Sidecar(
|
||||
version=version,
|
||||
classes=list(split.classes),
|
||||
input_size=a.input_size,
|
||||
backbone=a.backbone,
|
||||
mode=a.mode,
|
||||
trained_at=_now(),
|
||||
labels={"train": len(train), "val": len(val)},
|
||||
)
|
||||
tmp_side = out_dir / (SIDECAR_FILE + ".tmp")
|
||||
sidecar.write(tmp_side)
|
||||
onnx_pred = OnnxClassifier(tmp_model, tmp_side).predict_inputs(x_val.astype(np.float32)).argmax(axis=1)
|
||||
metrics.onnx_agreement = float((onnx_pred == np.array(y_pred)).mean()) if len(y_pred) else None
|
||||
sidecar.metrics = {
|
||||
"accuracy": metrics.accuracy,
|
||||
"macroRecall": metrics.macro_recall,
|
||||
"perClass": {c: m.__dict__ for c, m in metrics.per_class.items()},
|
||||
"floor": a.min_accuracy,
|
||||
}
|
||||
|
||||
written = metrics.accuracy >= a.min_accuracy and (metrics.onnx_agreement or 0.0) >= 0.99
|
||||
notes = []
|
||||
if metrics.onnx_agreement is not None and metrics.onnx_agreement < 0.99:
|
||||
notes.append(
|
||||
f"ONNX export disagrees with the torch model ({metrics.onnx_agreement:.3f}); model withheld"
|
||||
)
|
||||
report = render_report(
|
||||
version=version,
|
||||
trained_at=sidecar.trained_at,
|
||||
mode=a.mode,
|
||||
backbone=a.backbone,
|
||||
epochs=epochs,
|
||||
classes=split.classes,
|
||||
train_counts=split.counts("train"),
|
||||
val_counts=split.counts("val"),
|
||||
dropped=split.dropped,
|
||||
missing_files=missing,
|
||||
weights=weights,
|
||||
metrics=metrics,
|
||||
min_accuracy=a.min_accuracy,
|
||||
written=written,
|
||||
notes=notes,
|
||||
)
|
||||
(out_dir / REPORT_FILE).write_text(report)
|
||||
(out_dir / METRICS_FILE).write_text(json.dumps(metrics.to_dict(), indent=2) + "\n")
|
||||
if written:
|
||||
sidecar.write(out_dir / SIDECAR_FILE)
|
||||
tmp_model.replace(out_dir / MODEL_FILE)
|
||||
tmp_side.unlink(missing_ok=True)
|
||||
_log(
|
||||
f"MODEL WRITTEN: {out_dir / MODEL_FILE} "
|
||||
f"(accuracy {metrics.accuracy:.3f} >= floor {a.min_accuracy})"
|
||||
)
|
||||
else:
|
||||
tmp_model.unlink(missing_ok=True)
|
||||
tmp_side.unlink(missing_ok=True)
|
||||
_log(
|
||||
f"MODEL NOT WRITTEN: accuracy {metrics.accuracy:.3f} < floor {a.min_accuracy}; "
|
||||
f"see {out_dir / REPORT_FILE}"
|
||||
)
|
||||
print(report)
|
||||
return 0 if written else 3
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# evaluate — an existing model against labels that arrived AFTER it was trained, and its
|
||||
# view of the unlabelled pile (the ongoing accuracy check without labelling everything)
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def cmd_evaluate(a: argparse.Namespace) -> int:
|
||||
from .infer import OnnxClassifier
|
||||
|
||||
clf = OnnxClassifier(a.model)
|
||||
since = a.since or clf.sidecar.trained_at
|
||||
classes = clf.classes
|
||||
result: dict[str, object] = {"model": clf.sidecar.version, "classes": classes, "since": since}
|
||||
|
||||
reviewed = [s for s in load_reviewed_since(a.data, since) if s.label in classes]
|
||||
if reviewed:
|
||||
probs, kept = clf.predict_files([s.path for s in reviewed])
|
||||
rows = [reviewed[i] for i in kept]
|
||||
y_true = [classes.index(s.label) for s in rows]
|
||||
y_pred = probs.argmax(axis=1).tolist()
|
||||
m = compute_metrics(classes, y_true, y_pred, camera=[s.vision_class for s in rows])
|
||||
result["reviewedSince"] = m.to_dict()
|
||||
else:
|
||||
result["reviewedSince"] = None
|
||||
|
||||
pending = load_unlabelled(a.data, a.limit)
|
||||
if pending:
|
||||
probs, kept = clf.predict_files([s.path for s in pending])
|
||||
rows = [pending[i] for i in kept]
|
||||
pred = probs.argmax(axis=1)
|
||||
conf = probs.max(axis=1)
|
||||
hist = {c: int((pred == i).sum()) for i, c in enumerate(classes)}
|
||||
result["unlabelled"] = {
|
||||
"n": len(rows),
|
||||
"predicted": hist,
|
||||
"meanConfidence": float(conf.mean()) if len(rows) else None,
|
||||
"belowHalf": int((conf < 0.5).sum()),
|
||||
"agreesWithDetector": float(
|
||||
sum(1 for p, s in zip(pred, rows, strict=True) if classes[int(p)] == s.vision_class)
|
||||
/ len(rows)
|
||||
)
|
||||
if rows
|
||||
else None,
|
||||
}
|
||||
else:
|
||||
result["unlabelled"] = None
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# publish — the versioned files to a Gitea generic package (weights are not code, they
|
||||
# do not live in git; the vision image fetches them by URL at build)
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def cmd_publish(a: argparse.Namespace) -> int:
|
||||
d: Path = a.dir
|
||||
files = [d / MODEL_FILE, d / SIDECAR_FILE, d / REPORT_FILE, d / METRICS_FILE]
|
||||
for f in files[:2]:
|
||||
if not f.exists():
|
||||
_log(f"{f} missing — nothing to publish (a run below the floor writes no model)")
|
||||
return 1
|
||||
version = Sidecar.read(d / SIDECAR_FILE).version
|
||||
if not a.url:
|
||||
_log("no publish url: pass --url or set TRAINER_PUBLISH_URL")
|
||||
return 1
|
||||
token = a.token or os.environ.get("TRAINER_PUBLISH_TOKEN", "")
|
||||
if not token:
|
||||
_log("no token: pass --token or set TRAINER_PUBLISH_TOKEN")
|
||||
return 1
|
||||
base = a.url.rstrip("/") + "/" + version
|
||||
for f in files:
|
||||
if not f.exists():
|
||||
continue
|
||||
req = urllib.request.Request(f"{base}/{f.name}", data=f.read_bytes(), method="PUT")
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
req.add_header("Content-Type", "application/octet-stream")
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
_log(f"PUT {base}/{f.name} → {r.status}")
|
||||
_log(f"published {version}; pin it in apps/vision/models/bodytype.version and rebuild the vision image")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_serve(a: argparse.Namespace) -> int:
|
||||
from .server import serve
|
||||
|
||||
serve(
|
||||
a.data,
|
||||
a.out,
|
||||
a.host,
|
||||
a.port,
|
||||
os.environ.get("TRAINER_PUBLISH_URL", ""),
|
||||
os.environ.get("TRAINER_PUBLISH_TOKEN", ""),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="parking-trainer", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
def data_args(sp: argparse.ArgumentParser) -> None:
|
||||
sp.add_argument(
|
||||
"--data",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("TRAINER_DATA_DIR", "/data")),
|
||||
help="collector volume (collector.sqlite + crops/)",
|
||||
)
|
||||
|
||||
def split_args(sp: argparse.ArgumentParser) -> None:
|
||||
sp.add_argument(
|
||||
"--min-per-class",
|
||||
type=int,
|
||||
default=20,
|
||||
help="classes with fewer reviewed crops are dropped from the run",
|
||||
)
|
||||
sp.add_argument(
|
||||
"--val-fraction", type=float, default=0.2, help="newest fraction held out for validation"
|
||||
)
|
||||
|
||||
i = sub.add_parser("inspect", help="what a run would train on")
|
||||
data_args(i)
|
||||
split_args(i)
|
||||
i.set_defaults(fn=cmd_inspect)
|
||||
|
||||
t = sub.add_parser("train", help="train, evaluate, export (or refuse)")
|
||||
data_args(t)
|
||||
split_args(t)
|
||||
t.add_argument(
|
||||
"--out",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("TRAINER_OUT_DIR", "/out")),
|
||||
help="output root; a <version>/ folder is created under it",
|
||||
)
|
||||
t.add_argument("--mode", choices=["features", "finetune"], default="features")
|
||||
t.add_argument(
|
||||
"--backbone", choices=["resnet18", "mobilenet_v3_small", "efficientnet_b0"], default="resnet18"
|
||||
)
|
||||
t.add_argument("--epochs", type=int, default=0, help="0 = pick from the data size")
|
||||
t.add_argument("--batch", type=int, default=32)
|
||||
t.add_argument("--lr", type=float, default=1e-4, help="fine-tune learning rate")
|
||||
t.add_argument("--input-size", type=int, default=224)
|
||||
t.add_argument(
|
||||
"--min-accuracy", type=float, default=0.85, help="validation floor below which NO model is written"
|
||||
)
|
||||
t.add_argument("--version", default="", help="model version (default v<date>-<time>)")
|
||||
t.add_argument("--seed", type=int, default=7)
|
||||
t.add_argument(
|
||||
"--no-pretrained", action="store_true", help="random init (tests only — never for a real run)"
|
||||
)
|
||||
t.add_argument("--no-cache", action="store_true", help="do not read/write the feature cache")
|
||||
t.set_defaults(fn=cmd_train)
|
||||
|
||||
e = sub.add_parser(
|
||||
"evaluate", help="an existing model vs labels reviewed after it was trained + the unlabelled pile"
|
||||
)
|
||||
data_args(e)
|
||||
e.add_argument(
|
||||
"--model", type=Path, required=True, help="path to bodytype.onnx (sidecar .json beside it)"
|
||||
)
|
||||
e.add_argument("--since", default="", help="ISO time; default = the model's trained_at")
|
||||
e.add_argument(
|
||||
"--limit", type=int, default=2000, help="how many unlabelled crops to score (newest first)"
|
||||
)
|
||||
e.set_defaults(fn=cmd_evaluate)
|
||||
|
||||
u = sub.add_parser("publish", help="PUT a version folder to a Gitea generic package")
|
||||
u.add_argument("dir", type=Path, help="the <version>/ folder a passing run wrote")
|
||||
u.add_argument(
|
||||
"--url",
|
||||
default=os.environ.get("TRAINER_PUBLISH_URL", ""),
|
||||
help="https://<gitea>/api/packages/<owner>/generic/parking-bodytype (or TRAINER_PUBLISH_URL)",
|
||||
)
|
||||
u.add_argument("--token", default="", help="Gitea token with package:write (or TRAINER_PUBLISH_TOKEN)")
|
||||
u.set_defaults(fn=cmd_publish)
|
||||
|
||||
s = sub.add_parser("serve", help="the job API the collector's Training section talks to")
|
||||
data_args(s)
|
||||
s.add_argument("--out", type=Path, default=Path(os.environ.get("TRAINER_OUT_DIR", "/out")))
|
||||
s.add_argument("--host", default=os.environ.get("TRAINER_HOST", "0.0.0.0"))
|
||||
s.add_argument("--port", type=int, default=int(os.environ.get("TRAINER_PORT", "8091")))
|
||||
s.set_defaults(fn=cmd_serve)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
a = build_parser().parse_args(argv)
|
||||
try:
|
||||
return int(a.fn(a))
|
||||
except FileNotFoundError as exc:
|
||||
_log(str(exc))
|
||||
return 1
|
||||
|
||||
|
||||
__all__ = ["main", "build_parser", "VEHICLE_CLASSES", "CROP_MARGIN"]
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,179 @@
|
||||
"""The training set: reviewed, usable rows off the collector's SQLite, and how they are
|
||||
split and weighed. Pure Python + sqlite3 — no torch, so `inspect` and the tests run light.
|
||||
|
||||
Rules (wiki/decisions/bodytype-classifier-training.md):
|
||||
- Only rows a REVIEWER labelled count; the operator's pick and the camera's class are
|
||||
never labels. `unusable` rows are dropped.
|
||||
- Split by TIME (`at` = when the vehicle was seen): validation = the newest slice, so
|
||||
the number reflects tomorrow's traffic rather than a random shuffle of the same days.
|
||||
- Classes with too few labels are dropped from the run (and reported), never trained
|
||||
on a handful of examples that would make the softmax confidently wrong.
|
||||
- Class imbalance is weighed in the loss (inverse frequency, damped) and reported.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
# The shared vocabulary (packages/shared VEHICLE_CLASSES) — the only labels a reviewer can
|
||||
# give, and the only classes a model may emit. Order is the canonical one; the model's own
|
||||
# class list (sidecar) is the subset it trained on, in this order.
|
||||
VEHICLE_CLASSES: tuple[str, ...] = (
|
||||
"car",
|
||||
"sedan",
|
||||
"hatchback",
|
||||
"suv",
|
||||
"minivan",
|
||||
"pickup",
|
||||
"van",
|
||||
"truck",
|
||||
"bus",
|
||||
"motorcycle",
|
||||
)
|
||||
|
||||
DB_FILE = "collector.sqlite"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Sample:
|
||||
item: str
|
||||
booth: str
|
||||
kind: str # wash | entry
|
||||
at: str # ISO-8601, when the vehicle was seen (the split key)
|
||||
label: str # the reviewer's class
|
||||
vision_class: str # what the detector said (for the report, never a label)
|
||||
path: Path # the crop on disk
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Split:
|
||||
classes: tuple[str, ...]
|
||||
train: list[Sample]
|
||||
val: list[Sample]
|
||||
dropped: dict[str, int] # class → count, below the per-class minimum
|
||||
missing_files: int # labelled rows whose crop is not on disk
|
||||
|
||||
@property
|
||||
def class_index(self) -> dict[str, int]:
|
||||
return {c: i for i, c in enumerate(self.classes)}
|
||||
|
||||
def counts(self, part: str) -> dict[str, int]:
|
||||
rows = self.train if part == "train" else self.val
|
||||
c = Counter(s.label for s in rows)
|
||||
return {k: c.get(k, 0) for k in self.classes}
|
||||
|
||||
|
||||
_SELECT = "SELECT id, booth, kind, at, review_label, vision_class, image_path FROM items "
|
||||
|
||||
|
||||
def _rows(
|
||||
data_dir: Path, where: str, params: tuple[object, ...], db_file: Path | None = None
|
||||
) -> list[Sample]:
|
||||
db = db_file or data_dir / DB_FILE
|
||||
if not db.exists():
|
||||
raise FileNotFoundError(f"collector database not found: {db}")
|
||||
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||
try:
|
||||
rows = con.execute(_SELECT + where, params).fetchall()
|
||||
finally:
|
||||
con.close()
|
||||
out: list[Sample] = []
|
||||
for item, booth, kind, at, label, vision_class, image_path in rows:
|
||||
out.append(Sample(item, booth, kind, at, label or "", vision_class, data_dir / image_path))
|
||||
return out
|
||||
|
||||
|
||||
def load_labelled(data_dir: Path, db_file: Path | None = None) -> tuple[list[Sample], int]:
|
||||
"""Every reviewed, usable row with its crop path resolved. Returns (samples, missing)
|
||||
where `missing` counts rows whose crop file is gone (pruned/moved) — skipped."""
|
||||
rows = _rows(
|
||||
data_dir,
|
||||
"WHERE reviewed_at IS NOT NULL AND review_label != 'unusable' ORDER BY at, id",
|
||||
(),
|
||||
db_file,
|
||||
)
|
||||
out: list[Sample] = []
|
||||
missing = 0
|
||||
for s in rows:
|
||||
if s.label not in VEHICLE_CLASSES:
|
||||
continue # a label outside the vocabulary can only be a future/foreign row
|
||||
if not s.path.is_file():
|
||||
missing += 1
|
||||
continue
|
||||
out.append(s)
|
||||
return out, missing
|
||||
|
||||
|
||||
def load_reviewed_since(data_dir: Path, since: str, db_file: Path | None = None) -> list[Sample]:
|
||||
"""Usable labels whose REVIEW happened after `since` (ISO) — a clean held-out check for a
|
||||
model trained before then. Rows whose crop is gone are skipped."""
|
||||
rows = _rows(
|
||||
data_dir,
|
||||
"WHERE reviewed_at IS NOT NULL AND review_label != 'unusable' AND reviewed_at > ? "
|
||||
"ORDER BY reviewed_at, id",
|
||||
(since,),
|
||||
db_file,
|
||||
)
|
||||
return [s for s in rows if s.label in VEHICLE_CLASSES and s.path.is_file()]
|
||||
|
||||
|
||||
def load_unlabelled(data_dir: Path, limit: int = 2000, db_file: Path | None = None) -> list[Sample]:
|
||||
"""The pending pile, newest first — what the model would say about traffic nobody has
|
||||
labelled (its class histogram and detector agreement are the cheap drift check)."""
|
||||
rows = _rows(data_dir, "WHERE reviewed_at IS NULL ORDER BY received_at DESC LIMIT ?", (limit,), db_file)
|
||||
return [s for s in rows if s.path.is_file()]
|
||||
|
||||
|
||||
def make_split(samples: list[Sample], val_fraction: float = 0.2, min_per_class: int = 20) -> Split:
|
||||
"""Drop thin classes, then cut by time: the newest `val_fraction` is validation."""
|
||||
if not 0.0 < val_fraction < 1.0:
|
||||
raise ValueError("val_fraction must be in (0, 1)")
|
||||
counts = Counter(s.label for s in samples)
|
||||
kept = tuple(c for c in VEHICLE_CLASSES if counts.get(c, 0) >= min_per_class)
|
||||
dropped = {c: n for c, n in counts.items() if c not in kept}
|
||||
rows = sorted((s for s in samples if s.label in kept), key=lambda s: (s.at, s.item))
|
||||
n_val = int(round(len(rows) * val_fraction))
|
||||
if rows and n_val == 0:
|
||||
n_val = 1
|
||||
cut = len(rows) - n_val
|
||||
return Split(classes=kept, train=rows[:cut], val=rows[cut:], dropped=dropped, missing_files=0)
|
||||
|
||||
|
||||
def class_weights(split: Split, damping: float = 0.5) -> list[float]:
|
||||
"""Inverse-frequency weights for the loss, damped by `damping` (0.5 = square root, so a
|
||||
1:9 imbalance becomes 1:3 rather than 1:9 — full inverse weights over-correct on small
|
||||
sets). Normalised to mean 1 so the learning rate keeps its meaning."""
|
||||
counts = split.counts("train")
|
||||
total = sum(counts.values())
|
||||
k = len(split.classes)
|
||||
raw = [(total / (k * max(1, counts[c]))) ** damping for c in split.classes]
|
||||
mean = sum(raw) / max(1, len(raw))
|
||||
return [w / mean for w in raw]
|
||||
|
||||
|
||||
def summarise(samples: list[Sample]) -> dict[str, object]:
|
||||
"""What `inspect` prints: per-class counts, per-booth counts, time range."""
|
||||
by_class = Counter(s.label for s in samples)
|
||||
by_booth = Counter(s.booth for s in samples)
|
||||
by_kind = Counter(s.kind for s in samples)
|
||||
ats = sorted(s.at for s in samples)
|
||||
return {
|
||||
"total": len(samples),
|
||||
"byClass": {c: by_class.get(c, 0) for c in VEHICLE_CLASSES if by_class.get(c, 0)},
|
||||
"byBooth": dict(sorted(by_booth.items())),
|
||||
"byKind": dict(sorted(by_kind.items())),
|
||||
"from": ats[0] if ats else None,
|
||||
"to": ats[-1] if ats else None,
|
||||
}
|
||||
|
||||
|
||||
def suggested_epochs(n_train: int, mode: str) -> int:
|
||||
"""A sane default when the owner gives none: enough passes for a small set, fewer as
|
||||
it grows. Feature-extraction heads converge fast; fine-tunes need more but cost more."""
|
||||
if mode == "features":
|
||||
return int(min(200, max(30, 4000 / max(1, n_train) * 10)))
|
||||
return int(min(30, max(8, math.ceil(3000 / max(1, n_train)) * 4)))
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Run an exported classifier (ONNX + sidecar) — torch-free. Used by `evaluate` and by
|
||||
the tests; the vision service carries its own, equivalent, reader (vehicle.py)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .preprocess import Sidecar, load_input, softmax
|
||||
|
||||
|
||||
class OnnxClassifier:
|
||||
def __init__(self, model_path: Path, sidecar_path: Path | None = None) -> None:
|
||||
import onnxruntime as ort
|
||||
|
||||
self.model_path = Path(model_path)
|
||||
self.sidecar = Sidecar.read(sidecar_path or self.model_path.with_suffix(".json"))
|
||||
opts = ort.SessionOptions()
|
||||
opts.intra_op_num_threads = 2
|
||||
self._session = ort.InferenceSession(
|
||||
str(self.model_path), sess_options=opts, providers=["CPUExecutionProvider"]
|
||||
)
|
||||
self._input = self._session.get_inputs()[0].name
|
||||
|
||||
@property
|
||||
def classes(self) -> list[str]:
|
||||
return list(self.sidecar.classes)
|
||||
|
||||
def predict_inputs(self, x: Any, batch: int = 64) -> Any:
|
||||
"""[N,3,S,S] float32 → probabilities [N,K]."""
|
||||
import numpy as np
|
||||
|
||||
outs = []
|
||||
for i in range(0, len(x), batch):
|
||||
logits = self._session.run(None, {self._input: x[i : i + batch]})[0]
|
||||
outs.append(softmax(logits))
|
||||
return np.concatenate(outs, axis=0) if outs else np.zeros((0, len(self.classes)), np.float32)
|
||||
|
||||
def predict_files(self, paths: list[Path], batch: int = 64) -> tuple[Any, list[int]]:
|
||||
"""Decode + classify crop files. Returns (probs, indices of paths that decoded)."""
|
||||
import numpy as np
|
||||
|
||||
xs, kept = [], []
|
||||
for i, p in enumerate(paths):
|
||||
x = load_input(p, self.sidecar.input_size)
|
||||
if x is not None:
|
||||
xs.append(x)
|
||||
kept.append(i)
|
||||
if not xs:
|
||||
return np.zeros((0, len(self.classes)), np.float32), []
|
||||
return self.predict_inputs(np.stack(xs), batch), kept
|
||||
@@ -0,0 +1,305 @@
|
||||
"""The network and the two ways of training it. Imports torch — only `train` reaches here;
|
||||
everything else in the package stays torch-free (the `train` extra is heavy).
|
||||
|
||||
Backbone: a small ImageNet-pretrained torchvision model (BSD-3, weights included), used as
|
||||
a feature extractor; head: one linear layer over its pooled features.
|
||||
|
||||
- "features" mode: the backbone is FROZEN. Every crop goes through it once, the vectors
|
||||
are cached on disk (keyed by item id), and only the head is trained — minutes for a few
|
||||
thousand crops, seconds to retrain when new labels arrive. Expected to carry most of
|
||||
the accuracy on frontal gate views.
|
||||
- "finetune" mode: warm-starts the head the same way, then unfreezes everything and trains
|
||||
end to end with light augmentation. Roughly an hour on four Xeon cores for a few
|
||||
thousand crops with a mobile-sized backbone; the step when the cheap mode plateaus.
|
||||
|
||||
The exported ONNX graph takes raw RGB 0–255 float pixels and normalises INSIDE
|
||||
(see preprocess.py), so the vision service cannot get the constants wrong.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .data import Sample
|
||||
from .preprocess import IMAGENET_MEAN, IMAGENET_STD, load_input
|
||||
|
||||
BACKBONES: dict[str, int] = {"resnet18": 512, "mobilenet_v3_small": 576, "efficientnet_b0": 1280}
|
||||
|
||||
|
||||
def _torch() -> Any:
|
||||
import torch
|
||||
|
||||
torch.set_num_threads(max(1, os.cpu_count() or 1))
|
||||
return torch
|
||||
|
||||
|
||||
def build_backbone(name: str, pretrained: bool = True) -> Any:
|
||||
"""torchvision model with its classifier removed → pooled feature vector."""
|
||||
torch = _torch()
|
||||
import torchvision.models as tvm
|
||||
|
||||
if name not in BACKBONES:
|
||||
raise ValueError(f"unknown backbone {name!r} (choose from {', '.join(BACKBONES)})")
|
||||
if name == "resnet18":
|
||||
m = tvm.resnet18(weights=tvm.ResNet18_Weights.IMAGENET1K_V1 if pretrained else None)
|
||||
m.fc = torch.nn.Identity()
|
||||
elif name == "mobilenet_v3_small":
|
||||
m = tvm.mobilenet_v3_small(
|
||||
weights=tvm.MobileNet_V3_Small_Weights.IMAGENET1K_V1 if pretrained else None
|
||||
)
|
||||
m.classifier = torch.nn.Identity()
|
||||
else:
|
||||
m = tvm.efficientnet_b0(weights=tvm.EfficientNet_B0_Weights.IMAGENET1K_V1 if pretrained else None)
|
||||
m.classifier = torch.nn.Identity()
|
||||
return m
|
||||
|
||||
|
||||
class Classifier: # a factory, not a Module subclass at import time (torch is lazy)
|
||||
@staticmethod
|
||||
def make(backbone: Any, head: Any) -> Any:
|
||||
torch = _torch()
|
||||
|
||||
class _Net(torch.nn.Module): # type: ignore[misc,name-defined]
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.backbone = backbone
|
||||
self.head = head
|
||||
self.register_buffer("mean", torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1))
|
||||
self.register_buffer("std", torch.tensor(IMAGENET_STD).view(1, 3, 1, 1))
|
||||
|
||||
def forward(self, x: Any) -> Any:
|
||||
x = (x / 255.0 - self.mean) / self.std
|
||||
return self.head(self.backbone(x))
|
||||
|
||||
return _Net()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# Images in memory (uint8 — a few thousand 224² crops is a few hundred MB; float32 would
|
||||
# be four times that)
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_images(samples: list[Sample], input_size: int) -> tuple[Any, list[Sample]]:
|
||||
"""Decode + resize every crop once. Returns (uint8 [N,3,S,S], the samples that decoded)."""
|
||||
import numpy as np
|
||||
|
||||
xs, kept = [], []
|
||||
for s in samples:
|
||||
x = load_input(s.path, input_size)
|
||||
if x is None:
|
||||
continue
|
||||
xs.append(x.astype(np.uint8))
|
||||
kept.append(s)
|
||||
if not xs:
|
||||
return np.zeros((0, 3, input_size, input_size), np.uint8), []
|
||||
return np.stack(xs), kept
|
||||
|
||||
|
||||
def _batches(n: int, batch: int) -> list[slice]:
|
||||
return [slice(i, min(n, i + batch)) for i in range(0, n, batch)]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# Feature extraction (+ on-disk cache) and the head
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class FeatureCache:
|
||||
"""`<cache_dir>/features-<backbone>-<size>.npz`: item ids + vectors. Retraining the head
|
||||
after new labels arrive only runs the backbone on the NEW crops."""
|
||||
|
||||
path: Path
|
||||
|
||||
def load(self) -> dict[str, Any]:
|
||||
import numpy as np
|
||||
|
||||
if not self.path.exists():
|
||||
return {}
|
||||
z = np.load(self.path, allow_pickle=False)
|
||||
return dict(zip(z["ids"].tolist(), z["feats"], strict=True))
|
||||
|
||||
def save(self, table: dict[str, Any]) -> None:
|
||||
import numpy as np
|
||||
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
ids = np.array(list(table), dtype=str)
|
||||
feats = np.stack(list(table.values())) if table else np.zeros((0, 0), np.float32)
|
||||
np.savez(self.path, ids=ids, feats=feats)
|
||||
|
||||
|
||||
def extract_features(backbone: Any, x_u8: Any, batch: int = 64) -> Any:
|
||||
"""Frozen forward pass → [N,D] float32 (normalisation applied here, as in the graph)."""
|
||||
torch = _torch()
|
||||
import numpy as np
|
||||
|
||||
net = Classifier.make(backbone, torch.nn.Identity()).eval()
|
||||
out = []
|
||||
with torch.no_grad():
|
||||
for sl in _batches(len(x_u8), batch):
|
||||
xb = torch.from_numpy(x_u8[sl]).float()
|
||||
out.append(net(xb).numpy())
|
||||
return np.concatenate(out, axis=0) if out else np.zeros((0, 0), np.float32)
|
||||
|
||||
|
||||
def features_for(
|
||||
backbone: Any, samples: list[Sample], x_u8: Any, cache: FeatureCache | None, batch: int = 64
|
||||
) -> Any:
|
||||
"""Feature vectors for `samples` (aligned with x_u8), from the cache where present."""
|
||||
import numpy as np
|
||||
|
||||
table = cache.load() if cache else {}
|
||||
todo = [i for i, s in enumerate(samples) if s.item not in table]
|
||||
if todo:
|
||||
fresh = extract_features(backbone, x_u8[todo], batch)
|
||||
for i, f in zip(todo, fresh, strict=True):
|
||||
table[samples[i].item] = f.astype(np.float32)
|
||||
if cache:
|
||||
cache.save(table)
|
||||
return np.stack([table[s.item] for s in samples]) if samples else np.zeros((0, 0), np.float32)
|
||||
|
||||
|
||||
def train_head(
|
||||
feats: Any,
|
||||
y: list[int],
|
||||
n_classes: int,
|
||||
weights: list[float],
|
||||
epochs: int,
|
||||
lr: float = 1e-3,
|
||||
seed: int = 7,
|
||||
) -> Any:
|
||||
"""Multinomial logistic regression on cached features (full batch, Adam, weighted CE)."""
|
||||
torch = _torch()
|
||||
torch.manual_seed(seed)
|
||||
f = torch.from_numpy(feats).float()
|
||||
t = torch.tensor(y, dtype=torch.long)
|
||||
head = torch.nn.Linear(f.shape[1], n_classes)
|
||||
opt = torch.optim.Adam(head.parameters(), lr=lr, weight_decay=1e-4)
|
||||
loss_fn = torch.nn.CrossEntropyLoss(weight=torch.tensor(weights, dtype=torch.float32))
|
||||
head.train()
|
||||
for _ in range(epochs):
|
||||
opt.zero_grad()
|
||||
loss = loss_fn(head(f), t)
|
||||
loss.backward()
|
||||
opt.step()
|
||||
return head.eval()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# Full fine-tune
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _augment(xb: Any) -> Any:
|
||||
"""Light, label-preserving augmentation on a float batch [B,3,S,S] (0–255): horizontal
|
||||
flip (a gate view mirrored is still the same body type), a mild random zoom, and
|
||||
brightness/contrast jitter (dusk, headlights, wet tarmac)."""
|
||||
torch = _torch()
|
||||
b, _, s, _ = xb.shape
|
||||
flip = torch.rand(b) < 0.5
|
||||
xb = torch.where(flip.view(b, 1, 1, 1), xb.flip(-1), xb)
|
||||
# zoom: crop a random 85–100 % window and resize back
|
||||
out = torch.empty_like(xb)
|
||||
for i in range(b):
|
||||
frac = float(torch.empty(1).uniform_(0.85, 1.0))
|
||||
w = max(8, int(s * frac))
|
||||
x0 = int(torch.randint(0, s - w + 1, (1,)))
|
||||
y0 = int(torch.randint(0, s - w + 1, (1,)))
|
||||
crop = xb[i : i + 1, :, y0 : y0 + w, x0 : x0 + w]
|
||||
out[i : i + 1] = torch.nn.functional.interpolate(
|
||||
crop, size=(s, s), mode="bilinear", align_corners=False
|
||||
)
|
||||
bright = torch.empty(b, 1, 1, 1).uniform_(-25, 25)
|
||||
contrast = torch.empty(b, 1, 1, 1).uniform_(0.8, 1.2)
|
||||
mean = out.mean(dim=(1, 2, 3), keepdim=True)
|
||||
out = (out - mean) * contrast + mean + bright
|
||||
return out.clamp_(0, 255)
|
||||
|
||||
|
||||
def finetune(
|
||||
model: Any,
|
||||
x_u8: Any,
|
||||
y: list[int],
|
||||
weights: list[float],
|
||||
epochs: int,
|
||||
batch: int = 32,
|
||||
lr: float = 1e-4,
|
||||
seed: int = 7,
|
||||
log: Any = None,
|
||||
) -> Any:
|
||||
torch = _torch()
|
||||
import numpy as np
|
||||
|
||||
torch.manual_seed(seed)
|
||||
rng = np.random.default_rng(seed)
|
||||
t = torch.tensor(y, dtype=torch.long)
|
||||
opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-2)
|
||||
steps = epochs * max(1, (len(y) + batch - 1) // batch)
|
||||
sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=max(1, steps), pct_start=0.15)
|
||||
loss_fn = torch.nn.CrossEntropyLoss(weight=torch.tensor(weights, dtype=torch.float32))
|
||||
for epoch in range(epochs):
|
||||
model.train()
|
||||
order = rng.permutation(len(y))
|
||||
total = 0.0
|
||||
for sl in _batches(len(y), batch):
|
||||
idx = order[sl]
|
||||
xb = _augment(torch.from_numpy(x_u8[idx]).float())
|
||||
opt.zero_grad()
|
||||
loss = loss_fn(model(xb), t[idx])
|
||||
loss.backward()
|
||||
opt.step()
|
||||
sched.step()
|
||||
total += float(loss.detach()) * len(idx)
|
||||
if log:
|
||||
log(f"epoch {epoch + 1}/{epochs} loss {total / max(1, len(y)):.4f}")
|
||||
return model.eval()
|
||||
|
||||
|
||||
def predict_logits(model: Any, x_u8: Any, batch: int = 64) -> Any:
|
||||
torch = _torch()
|
||||
import numpy as np
|
||||
|
||||
model.eval()
|
||||
out = []
|
||||
with torch.no_grad():
|
||||
for sl in _batches(len(x_u8), batch):
|
||||
out.append(model(torch.from_numpy(x_u8[sl]).float()).numpy())
|
||||
return np.concatenate(out, axis=0) if out else np.zeros((0, 0), np.float32)
|
||||
|
||||
|
||||
def export_onnx(model: Any, input_size: int, path: Path) -> None:
|
||||
"""Export with the current (torch.export-based) exporter; fall back to the legacy
|
||||
TorchScript one where the new path is unavailable or trips over an op. Whichever wrote
|
||||
the graph, the job then checks it against the torch model (onnx_agreement) before the
|
||||
file is kept."""
|
||||
torch = _torch()
|
||||
|
||||
model.eval()
|
||||
dummy = torch.zeros(1, 3, input_size, input_size)
|
||||
names: dict[str, Any] = dict(input_names=["image"], output_names=["logits"])
|
||||
try:
|
||||
torch.onnx.export(
|
||||
model,
|
||||
(dummy,),
|
||||
str(path),
|
||||
dynamo=True,
|
||||
dynamic_shapes={"x": {0: "batch"}},
|
||||
opset_version=18,
|
||||
external_data=False, # ONE file: the vision image bakes bodytype.onnx + .json, nothing else
|
||||
verbose=False,
|
||||
**names,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - any failure → the legacy exporter
|
||||
torch.onnx.export(
|
||||
model,
|
||||
dummy,
|
||||
str(path),
|
||||
dynamo=False,
|
||||
dynamic_axes={"image": {0: "batch"}, "logits": {0: "batch"}},
|
||||
opset_version=17,
|
||||
**names,
|
||||
)
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Crop → model input. THE CONTRACT between the trainer and the vision service's classifier
|
||||
stage: what the network sees at training time must be exactly what it sees on the booth.
|
||||
|
||||
The trainer does not share code with the vision service (different packages, different
|
||||
images), so the contract is DATA: every constant here is written into the model's sidecar
|
||||
(`bodytype.json`) and the vision side reads and applies them from there — nothing is
|
||||
assumed on either side. Both use OpenCV with the same interpolation so the pixels match.
|
||||
|
||||
- input: the collector's crop (the detector's vehicle box + margin, plate blurred), or on
|
||||
the booth the same cut made live from the frame (vehicle.py mirrors `makeReviewCrop`).
|
||||
- resize: squash to input_size × input_size with INTER_AREA (the crop IS the vehicle; no
|
||||
centre-crop that would lose a bumper or a roofline — the shape is the signal).
|
||||
- colour: RGB, float32, 0–255. Normalisation (/255, ImageNet mean/std) lives INSIDE the
|
||||
ONNX graph, so a consumer feeds raw pixels and cannot get the constants wrong.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SIDECAR_FORMAT = "parking-bodytype/1"
|
||||
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
||||
IMAGENET_STD = (0.229, 0.224, 0.225)
|
||||
CROP_MARGIN = 0.08 # must equal CROP_MARGIN in apps/server review-outbox.ts
|
||||
|
||||
|
||||
@dataclass
|
||||
class Sidecar:
|
||||
"""`bodytype.json` beside `bodytype.onnx`."""
|
||||
|
||||
version: str
|
||||
classes: list[str]
|
||||
input_size: int = 224
|
||||
color: str = "rgb"
|
||||
resize: str = "area"
|
||||
crop_margin: float = CROP_MARGIN
|
||||
normalization: str = "in-graph" # the ONNX divides by 255 and applies mean/std itself
|
||||
mean: list[float] = field(default_factory=lambda: list(IMAGENET_MEAN))
|
||||
std: list[float] = field(default_factory=lambda: list(IMAGENET_STD))
|
||||
backbone: str = ""
|
||||
mode: str = ""
|
||||
trained_at: str = ""
|
||||
labels: dict[str, int] = field(default_factory=dict) # train / val counts
|
||||
metrics: dict[str, Any] = field(default_factory=dict) # accuracy, macro, per-class
|
||||
format: str = SIDECAR_FORMAT
|
||||
|
||||
def write(self, path: Path) -> None:
|
||||
path.write_text(json.dumps(asdict(self), indent=2) + "\n")
|
||||
|
||||
@classmethod
|
||||
def read(cls, path: Path) -> Sidecar:
|
||||
d = json.loads(path.read_text())
|
||||
if d.get("format") != SIDECAR_FORMAT:
|
||||
raise ValueError(f"{path}: unknown sidecar format {d.get('format')!r}")
|
||||
known = {f for f in cls.__dataclass_fields__}
|
||||
return cls(**{k: v for k, v in d.items() if k in known})
|
||||
|
||||
|
||||
def load_input(path: Path, input_size: int) -> Any:
|
||||
"""Decode a crop and produce the network input: RGB float32 CHW, 0–255, squashed to
|
||||
input_size. Returns None when the file cannot be decoded."""
|
||||
import cv2
|
||||
|
||||
img = cv2.imread(str(path), cv2.IMREAD_COLOR)
|
||||
if img is None:
|
||||
return None
|
||||
return array_to_input(img, input_size)
|
||||
|
||||
|
||||
def array_to_input(bgr: Any, input_size: int) -> Any:
|
||||
"""BGR uint8 HWC (OpenCV's native) → RGB float32 CHW 0–255 at input_size."""
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
resized = cv2.resize(bgr, (input_size, input_size), interpolation=cv2.INTER_AREA)
|
||||
rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)
|
||||
return np.ascontiguousarray(rgb.transpose(2, 0, 1).astype(np.float32))
|
||||
|
||||
|
||||
def softmax(logits: Any) -> Any:
|
||||
import numpy as np
|
||||
|
||||
z = logits - logits.max(axis=-1, keepdims=True)
|
||||
e = np.exp(z)
|
||||
return e / e.sum(axis=-1, keepdims=True)
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Metrics + the human-readable report. The owner reads this BEFORE anything ships; the
|
||||
floor decision is made on these numbers. Pure Python, no torch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassMetrics:
|
||||
support: int
|
||||
recall: float # of the true members, how many the model caught
|
||||
precision: float # of the model's picks, how many were right
|
||||
|
||||
|
||||
@dataclass
|
||||
class Metrics:
|
||||
n: int
|
||||
accuracy: float
|
||||
macro_recall: float
|
||||
per_class: dict[str, ClassMetrics]
|
||||
confusion: list[list[int]] # rows = true class, cols = predicted, in `classes` order
|
||||
camera_agreement: float | None = None # how often the model equals the detector's class
|
||||
onnx_agreement: float | None = None # exported graph vs the torch model, argmax
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def compute_metrics(
|
||||
classes: tuple[str, ...] | list[str],
|
||||
y_true: list[int],
|
||||
y_pred: list[int],
|
||||
camera: list[str] | None = None,
|
||||
) -> Metrics:
|
||||
k = len(classes)
|
||||
conf = [[0] * k for _ in range(k)]
|
||||
for t, p in zip(y_true, y_pred, strict=True):
|
||||
conf[t][p] += 1
|
||||
per: dict[str, ClassMetrics] = {}
|
||||
recalls: list[float] = []
|
||||
for i, c in enumerate(classes):
|
||||
support = sum(conf[i])
|
||||
tp = conf[i][i]
|
||||
picked = sum(conf[r][i] for r in range(k))
|
||||
recall = tp / support if support else 0.0
|
||||
precision = tp / picked if picked else 0.0
|
||||
per[c] = ClassMetrics(support=support, recall=recall, precision=precision)
|
||||
if support:
|
||||
recalls.append(recall)
|
||||
n = len(y_true)
|
||||
acc = sum(1 for t, p in zip(y_true, y_pred, strict=True) if t == p) / n if n else 0.0
|
||||
agree = None
|
||||
if camera is not None and n:
|
||||
agree = sum(1 for p, cam in zip(y_pred, camera, strict=True) if classes[p] == cam) / n
|
||||
return Metrics(
|
||||
n=n,
|
||||
accuracy=acc,
|
||||
macro_recall=sum(recalls) / len(recalls) if recalls else 0.0,
|
||||
per_class=per,
|
||||
confusion=conf,
|
||||
camera_agreement=agree,
|
||||
)
|
||||
|
||||
|
||||
def _pct(x: float | None) -> str:
|
||||
return "—" if x is None else f"{x * 100:.1f} %"
|
||||
|
||||
|
||||
def render_report(
|
||||
*,
|
||||
version: str,
|
||||
trained_at: str,
|
||||
mode: str,
|
||||
backbone: str,
|
||||
epochs: int,
|
||||
classes: tuple[str, ...] | list[str],
|
||||
train_counts: dict[str, int],
|
||||
val_counts: dict[str, int],
|
||||
dropped: dict[str, int],
|
||||
missing_files: int,
|
||||
weights: list[float],
|
||||
metrics: Metrics,
|
||||
min_accuracy: float,
|
||||
written: bool,
|
||||
notes: list[str] | None = None,
|
||||
) -> str:
|
||||
lines: list[str] = []
|
||||
verdict = "MODEL WRITTEN" if written else "MODEL NOT WRITTEN — below the floor"
|
||||
lines.append(f"# Body-type classifier {version}")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
f"**{verdict}** · validation accuracy {_pct(metrics.accuracy)} vs floor {_pct(min_accuracy)}"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append(f"- trained: {trained_at}")
|
||||
lines.append(f"- mode: {mode} · backbone: {backbone} · epochs: {epochs}")
|
||||
lines.append(f"- classes ({len(classes)}): {', '.join(classes)}")
|
||||
lines.append(
|
||||
f"- labels: {sum(train_counts.values())} train · {sum(val_counts.values())} validation "
|
||||
"(validation = the NEWEST slice, by time seen)"
|
||||
)
|
||||
if dropped:
|
||||
lines.append(
|
||||
"- dropped (too few labels this run): "
|
||||
+ ", ".join(f"{c} ({n})" for c, n in sorted(dropped.items()))
|
||||
)
|
||||
if missing_files:
|
||||
lines.append(f"- labelled rows whose crop is missing on disk (skipped): {missing_files}")
|
||||
lines.append("")
|
||||
lines.append("## Validation")
|
||||
lines.append("")
|
||||
lines.append(f"- accuracy: {_pct(metrics.accuracy)} on {metrics.n} crops")
|
||||
lines.append(f"- macro recall (each class counted equally): {_pct(metrics.macro_recall)}")
|
||||
if metrics.camera_agreement is not None:
|
||||
lines.append(
|
||||
f"- agrees with the detector's coarse class: {_pct(metrics.camera_agreement)} "
|
||||
"(informational — the detector only knows car/truck/bus/motorcycle)"
|
||||
)
|
||||
if metrics.onnx_agreement is not None:
|
||||
lines.append(
|
||||
f"- exported ONNX matches the trained model on validation: {_pct(metrics.onnx_agreement)}"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("| class | train | val | recall | precision | loss weight |")
|
||||
lines.append("|---|---:|---:|---:|---:|---:|")
|
||||
for c, w in zip(classes, weights, strict=True):
|
||||
m = metrics.per_class[c]
|
||||
lines.append(
|
||||
f"| {c} | {train_counts.get(c, 0)} | {m.support} | {_pct(m.recall) if m.support else '—'} | "
|
||||
f"{_pct(m.precision) if m.support else '—'} | {w:.2f} |"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("## Confusion (rows = reviewer's label, columns = model)")
|
||||
lines.append("")
|
||||
lines.append("| | " + " | ".join(classes) + " |")
|
||||
lines.append("|---|" + "---:|" * len(classes))
|
||||
for c, row in zip(classes, metrics.confusion, strict=True):
|
||||
lines.append(f"| **{c}** | " + " | ".join(str(n) for n in row) + " |")
|
||||
lines.append("")
|
||||
if notes:
|
||||
lines.append("## Notes")
|
||||
lines.append("")
|
||||
lines.extend(f"- {n}" for n in notes)
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"The flag on the booth records, it never bills: even a model that passes the floor is "
|
||||
"advisory (the site threshold gates the flag)."
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -0,0 +1,402 @@
|
||||
"""`parking-trainer serve` — the job API behind the collector's Training section.
|
||||
|
||||
A tiny stdlib HTTP server (no framework, no extra deps) on the compose-internal network,
|
||||
never published: the collector proxies to it behind the reviewer's login. One job at a
|
||||
time; each job is the CLI run as a SUBPROCESS (`python -m trainer.cli …`) with its output
|
||||
captured to a log file — torch's memory goes away with the process, and a crashing job
|
||||
cannot take the service down. Job state + logs persist under `<out>/jobs/` so a restart
|
||||
still shows history.
|
||||
|
||||
GET /health {ok, busy, version}
|
||||
GET /readiness what `inspect` prints (+ the defaults the UI offers)
|
||||
GET /versions every <out>/<version>/ folder: written?, metrics, sidecar
|
||||
GET /versions/<v>/report report.md (text/markdown)
|
||||
GET /jobs recent jobs, newest first
|
||||
GET /jobs/<id> one job incl. the log tail
|
||||
POST /jobs {kind: train|evaluate|publish, …args} → 202 {id} | 409 busy
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .cli import METRICS_FILE, MODEL_FILE, REPORT_FILE, SIDECAR_FILE
|
||||
from .data import load_labelled, make_split, summarise
|
||||
from .preprocess import Sidecar
|
||||
|
||||
_VERSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
||||
BACKBONES = ("resnet18", "mobilenet_v3_small", "efficientnet_b0")
|
||||
MODES = ("features", "finetune")
|
||||
LOG_TAIL_BYTES = 16_000
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
class Jobs:
|
||||
"""The single-slot job runner. `start` refuses while one runs."""
|
||||
|
||||
def __init__(self, data_dir: Path, out_dir: Path, publish_url: str, publish_token: str) -> None:
|
||||
self.data_dir = data_dir
|
||||
self.out_dir = out_dir
|
||||
self.publish_url = publish_url
|
||||
self.publish_token = publish_token
|
||||
self.jobs_dir = out_dir / "jobs"
|
||||
self.jobs_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._lock = threading.Lock()
|
||||
self._current: dict[str, Any] | None = None
|
||||
self._proc: subprocess.Popen[bytes] | None = None
|
||||
|
||||
# ---- state -----------------------------------------------------------------------
|
||||
def _write(self, job: dict[str, Any]) -> None:
|
||||
(self.jobs_dir / f"{job['id']}.json").write_text(json.dumps(job, indent=2))
|
||||
|
||||
def _read(self, job_id: str) -> dict[str, Any] | None:
|
||||
p = self.jobs_dir / f"{job_id}.json"
|
||||
if not p.is_file():
|
||||
return None
|
||||
return json.loads(p.read_text()) # type: ignore[no-any-return]
|
||||
|
||||
def log_tail(self, job_id: str) -> str:
|
||||
p = self.jobs_dir / f"{job_id}.log"
|
||||
if not p.is_file():
|
||||
return ""
|
||||
size = p.stat().st_size
|
||||
with p.open("rb") as f:
|
||||
if size > LOG_TAIL_BYTES:
|
||||
f.seek(size - LOG_TAIL_BYTES)
|
||||
return f.read().decode("utf-8", "replace")
|
||||
|
||||
@property
|
||||
def busy(self) -> bool:
|
||||
return self._current is not None
|
||||
|
||||
def current(self) -> dict[str, Any] | None:
|
||||
return dict(self._current) if self._current else None
|
||||
|
||||
def get(self, job_id: str) -> dict[str, Any] | None:
|
||||
if self._current and self._current["id"] == job_id:
|
||||
job = dict(self._current)
|
||||
else:
|
||||
job = self._read(job_id) or {}
|
||||
if not job:
|
||||
return None
|
||||
job["log"] = self.log_tail(job_id)
|
||||
return job
|
||||
|
||||
def recent(self, limit: int = 20) -> list[dict[str, Any]]:
|
||||
files = sorted(self.jobs_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
out = []
|
||||
for p in files[:limit]:
|
||||
try:
|
||||
out.append(json.loads(p.read_text()))
|
||||
except ValueError:
|
||||
continue
|
||||
if self._current and all(j["id"] != self._current["id"] for j in out):
|
||||
out.insert(0, dict(self._current))
|
||||
out.sort(key=lambda j: j.get("startedAt", ""), reverse=True)
|
||||
return out
|
||||
|
||||
# ---- args → CLI ------------------------------------------------------------------
|
||||
def _argv(self, kind: str, a: dict[str, Any]) -> list[str]:
|
||||
base = [sys.executable, "-m", "trainer.cli"]
|
||||
if kind == "train":
|
||||
mode = a.get("mode", "features")
|
||||
backbone = a.get("backbone", "resnet18")
|
||||
if mode not in MODES or backbone not in BACKBONES:
|
||||
raise ValueError("bad mode/backbone")
|
||||
floor = float(a.get("minAccuracy", 0.85))
|
||||
min_per = int(a.get("minPerClass", 20))
|
||||
epochs = int(a.get("epochs", 0))
|
||||
if not 0.0 <= floor <= 1.0 or min_per < 1 or epochs < 0:
|
||||
raise ValueError("bad numbers")
|
||||
argv = base + [
|
||||
"train",
|
||||
"--data",
|
||||
str(self.data_dir),
|
||||
"--out",
|
||||
str(self.out_dir),
|
||||
"--mode",
|
||||
mode,
|
||||
"--backbone",
|
||||
backbone,
|
||||
"--min-accuracy",
|
||||
str(floor),
|
||||
"--min-per-class",
|
||||
str(min_per),
|
||||
"--epochs",
|
||||
str(epochs),
|
||||
]
|
||||
if a.get("version"):
|
||||
argv += ["--version", self._version(a["version"])]
|
||||
return argv
|
||||
if kind == "evaluate":
|
||||
v = self._version(a.get("version", ""))
|
||||
return base + [
|
||||
"evaluate",
|
||||
"--data",
|
||||
str(self.data_dir),
|
||||
"--model",
|
||||
str(self.out_dir / v / MODEL_FILE),
|
||||
]
|
||||
if kind == "publish":
|
||||
v = self._version(a.get("version", ""))
|
||||
url = str(a.get("url") or self.publish_url)
|
||||
if not url.startswith("https://") and not url.startswith("http://"):
|
||||
raise ValueError("bad publish url")
|
||||
return base + ["publish", str(self.out_dir / v), "--url", url]
|
||||
raise ValueError("kind must be train, evaluate or publish")
|
||||
|
||||
@staticmethod
|
||||
def _version(v: Any) -> str:
|
||||
if not isinstance(v, str) or not _VERSION_RE.match(v) or v in ("cache", "jobs"):
|
||||
raise ValueError("bad version")
|
||||
return v
|
||||
|
||||
# ---- run -------------------------------------------------------------------------
|
||||
def start(self, kind: str, args: dict[str, Any]) -> dict[str, Any]:
|
||||
argv = self._argv(kind, args)
|
||||
with self._lock:
|
||||
if self._current is not None:
|
||||
raise RuntimeError("busy")
|
||||
job_id = f"{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}-{uuid.uuid4().hex[:6]}"
|
||||
job: dict[str, Any] = {
|
||||
"id": job_id,
|
||||
"kind": kind,
|
||||
"args": {k: v for k, v in args.items() if k != "token"},
|
||||
"status": "running",
|
||||
"startedAt": _now(),
|
||||
"finishedAt": None,
|
||||
"exitCode": None,
|
||||
}
|
||||
env = dict(os.environ)
|
||||
if kind == "publish":
|
||||
env["TRAINER_PUBLISH_TOKEN"] = self.publish_token
|
||||
log = (self.jobs_dir / f"{job_id}.log").open("wb")
|
||||
log.write(f"$ {' '.join(argv[3:])}\n".encode())
|
||||
log.flush()
|
||||
self._proc = subprocess.Popen(argv, stdout=log, stderr=subprocess.STDOUT, env=env)
|
||||
self._current = job
|
||||
self._write(job)
|
||||
threading.Thread(target=self._wait, args=(job, log), daemon=True).start()
|
||||
return dict(job)
|
||||
|
||||
def _wait(self, job: dict[str, Any], log: Any) -> None:
|
||||
assert self._proc is not None
|
||||
code = self._proc.wait()
|
||||
log.close()
|
||||
with self._lock:
|
||||
job["exitCode"] = code
|
||||
job["finishedAt"] = _now()
|
||||
job["status"] = "done" if code == 0 else ("refused" if code in (2, 3) else "failed")
|
||||
self._write(job)
|
||||
self._current = None
|
||||
self._proc = None
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def readiness(data_dir: Path, min_per_class: int = 20, val_fraction: float = 0.2) -> dict[str, Any]:
|
||||
try:
|
||||
samples, missing = load_labelled(data_dir)
|
||||
except FileNotFoundError:
|
||||
return {
|
||||
"ready": False,
|
||||
"labelled": {"total": 0, "byClass": {}},
|
||||
"missingCrops": 0,
|
||||
"error": "no collector database yet",
|
||||
}
|
||||
split = make_split(samples, val_fraction, min_per_class)
|
||||
return {
|
||||
"ready": len(split.classes) >= 2,
|
||||
"labelled": summarise(samples),
|
||||
"missingCrops": missing,
|
||||
"run": {
|
||||
"classes": list(split.classes),
|
||||
"train": split.counts("train"),
|
||||
"val": split.counts("val"),
|
||||
"dropped": split.dropped,
|
||||
"minPerClass": min_per_class,
|
||||
"valFraction": val_fraction,
|
||||
},
|
||||
"defaults": {
|
||||
"mode": "features",
|
||||
"backbone": "resnet18",
|
||||
"minAccuracy": 0.85,
|
||||
"minPerClass": min_per_class,
|
||||
},
|
||||
"modes": list(MODES),
|
||||
"backbones": list(BACKBONES),
|
||||
}
|
||||
|
||||
|
||||
def versions(out_dir: Path) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
if not out_dir.is_dir():
|
||||
return out
|
||||
for d in sorted(out_dir.iterdir(), key=lambda p: p.name, reverse=True):
|
||||
if not d.is_dir() or d.name in ("cache", "jobs"):
|
||||
continue
|
||||
if not (d / REPORT_FILE).is_file() and not (d / METRICS_FILE).is_file():
|
||||
continue
|
||||
entry: dict[str, Any] = {
|
||||
"version": d.name,
|
||||
"written": (d / MODEL_FILE).is_file() and (d / SIDECAR_FILE).is_file(),
|
||||
"hasReport": (d / REPORT_FILE).is_file(),
|
||||
"modifiedAt": datetime.fromtimestamp(d.stat().st_mtime, tz=timezone.utc)
|
||||
.replace(microsecond=0)
|
||||
.isoformat(),
|
||||
}
|
||||
try:
|
||||
m = json.loads((d / METRICS_FILE).read_text())
|
||||
entry["accuracy"] = m.get("accuracy")
|
||||
entry["macroRecall"] = m.get("macro_recall")
|
||||
entry["n"] = m.get("n")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
if entry["written"]:
|
||||
try:
|
||||
side = Sidecar.read(d / SIDECAR_FILE)
|
||||
entry["classes"] = side.classes
|
||||
entry["mode"] = side.mode
|
||||
entry["backbone"] = side.backbone
|
||||
entry["trainedAt"] = side.trained_at
|
||||
entry["labels"] = side.labels
|
||||
entry["floor"] = side.metrics.get("floor")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
jobs: Jobs # set on the class by serve()
|
||||
server_version = "parking-trainer"
|
||||
|
||||
def log_message(self, fmt: str, *args: Any) -> None: # quieter than the default
|
||||
sys.stderr.write(f"[trainer.serve] {self.address_string()} {fmt % args}\n")
|
||||
|
||||
def _json(self, code: int, body: Any) -> None:
|
||||
raw = json.dumps(body).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
def _text(self, code: int, body: str, ctype: str = "text/markdown; charset=utf-8") -> None:
|
||||
raw = body.encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
self._guarded(self._get)
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
self._guarded(self._post)
|
||||
|
||||
def _guarded(self, handler: Any) -> None:
|
||||
"""Run a handler; an unexpected exception becomes a 500 JSON reply instead of a
|
||||
dropped connection (the stdlib server would print the traceback and close the
|
||||
socket, which the collector could only report as "fetch failed" — 2026-09-16,
|
||||
a collector DB from before the `kind` column)."""
|
||||
try:
|
||||
handler()
|
||||
except Exception as exc: # noqa: BLE001 — anything: the reply must be a reply
|
||||
sys.stderr.write(f"[trainer.serve] {self.command} {self.path}: {type(exc).__name__}: {exc}\n")
|
||||
try:
|
||||
self._json(500, {"error": f"{type(exc).__name__}: {exc}"})
|
||||
except Exception: # noqa: BLE001 — headers already sent; nothing left to do
|
||||
pass
|
||||
|
||||
def _get(self) -> None:
|
||||
path = self.path.split("?", 1)[0]
|
||||
j = self.jobs
|
||||
if path == "/health":
|
||||
self._json(200, {"ok": True, "busy": j.busy, "version": "parking-trainer"})
|
||||
elif path == "/readiness":
|
||||
self._json(200, readiness(j.data_dir))
|
||||
elif path == "/versions":
|
||||
self._json(200, {"versions": versions(j.out_dir)})
|
||||
elif path.startswith("/versions/") and path.endswith("/report"):
|
||||
v = path[len("/versions/") : -len("/report")]
|
||||
try:
|
||||
p = j.out_dir / Jobs._version(v) / REPORT_FILE
|
||||
except ValueError:
|
||||
self._json(400, {"error": "bad version"})
|
||||
return
|
||||
if not p.is_file():
|
||||
self._json(404, {"error": "no report"})
|
||||
else:
|
||||
self._text(200, p.read_text())
|
||||
elif path == "/jobs":
|
||||
self._json(200, {"jobs": j.recent(), "current": j.current()})
|
||||
elif path.startswith("/jobs/"):
|
||||
job = j.get(path[len("/jobs/") :])
|
||||
self._json(200, job) if job else self._json(404, {"error": "no such job"})
|
||||
else:
|
||||
self._json(404, {"error": "not found"})
|
||||
|
||||
def _post(self) -> None:
|
||||
if self.path.split("?", 1)[0] != "/jobs":
|
||||
self._json(404, {"error": "not found"})
|
||||
return
|
||||
n = int(self.headers.get("Content-Length") or 0)
|
||||
if n > 64_000:
|
||||
self._json(413, {"error": "too large"})
|
||||
return
|
||||
try:
|
||||
body = json.loads(self.rfile.read(n) or b"{}")
|
||||
if not isinstance(body, dict):
|
||||
raise ValueError("object expected")
|
||||
except ValueError as exc:
|
||||
self._json(400, {"error": f"bad json: {exc}"})
|
||||
return
|
||||
kind = str(body.pop("kind", ""))
|
||||
try:
|
||||
job = self.jobs.start(kind, body)
|
||||
except ValueError as exc:
|
||||
self._json(400, {"error": str(exc)})
|
||||
return
|
||||
except RuntimeError:
|
||||
self._json(409, {"error": "a job is already running", "current": self.jobs.current()})
|
||||
return
|
||||
self._json(202, job)
|
||||
|
||||
|
||||
def serve(data_dir: Path, out_dir: Path, host: str, port: int, publish_url: str, publish_token: str) -> None:
|
||||
Handler.jobs = Jobs(data_dir, out_dir, publish_url, publish_token)
|
||||
httpd = ThreadingHTTPServer((host, port), Handler)
|
||||
sys.stderr.write(f"[trainer.serve] listening on {host}:{port}, data {data_dir}, out {out_dir}\n")
|
||||
try:
|
||||
httpd.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
httpd.server_close()
|
||||
|
||||
|
||||
def wait_idle(jobs: Jobs, timeout: float = 60.0) -> None:
|
||||
"""Test helper: block until no job runs."""
|
||||
t0 = time.monotonic()
|
||||
while jobs.busy and time.monotonic() - t0 < timeout:
|
||||
time.sleep(0.1)
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"$schema": "https://turbo.build/schema.json",
|
||||
"extends": ["//"],
|
||||
"tasks": {
|
||||
"build": {
|
||||
"outputs": []
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
+1444
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,9 @@
|
||||
.venv/
|
||||
**/__pycache__/
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.env
|
||||
# weights are fetched inside the build (yolox) or pinned by models/bodytype.version
|
||||
models/*
|
||||
!models/bodytype.version
|
||||
@@ -29,3 +29,11 @@ VISION_MIN_CONFIDENCE=0.5
|
||||
# VISION_VEHICLE_MODEL_PATH=models/yolox_s.onnx
|
||||
# VISION_VEHICLE_INPUT_SIZE=640
|
||||
# VISION_VEHICLE_MIN_CONFIDENCE=0.4
|
||||
|
||||
# Phase B: the body-type classifier (sedan/hatchback/suv/… on the detector's crop), trained
|
||||
# by apps/trainer on the reviewer's labels. The Docker image bakes it at
|
||||
# /app/models/bodytype.onnx (+ .json sidecar) when models/bodytype.version pins a published
|
||||
# version; locally copy a trainer output folder's two files into apps/vision/models/.
|
||||
# Path set but no file = stage off (the normal state before the first model).
|
||||
# VISION_VEHICLE_CLASSIFIER_PATH=models/bodytype.onnx
|
||||
# VISION_VEHICLE_CLASSIFIER_MIN_CONFIDENCE=0.6
|
||||
|
||||
@@ -7,5 +7,6 @@ __pycache__/
|
||||
.ruff_cache/
|
||||
|
||||
# Model weights (fetched at deploy / first run, never committed — can be large + license-scoped)
|
||||
models/
|
||||
models/*
|
||||
!models/bodytype.version
|
||||
*.onnx
|
||||
|
||||
+20
-1
@@ -33,6 +33,24 @@ ARG YOLOX_URL=https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.
|
||||
RUN mkdir -p /app/models \
|
||||
&& (curl -fsSL -o /app/models/yolox_s.onnx "$YOLOX_URL" \
|
||||
|| (echo "[build] yolox weights not fetched (no network) — vehicle stage off" && rm -f /app/models/yolox_s.onnx))
|
||||
# Phase B body-type classifier (apps/trainer output, published to the Gitea generic package
|
||||
# registry — weights are not code, they never live in git). models/bodytype.version PINS the
|
||||
# version this image carries: empty = no classifier (phase B off). A pinned version that
|
||||
# cannot be fetched FAILS the build — the image must carry what git says it carries. The
|
||||
# registry may need auth: pass a BuildKit secret `bodytype_auth` holding "user:token".
|
||||
ARG BODYTYPE_BASE_URL=https://git.infra.msai.al/api/packages/mca/generic/parking-bodytype
|
||||
COPY models/bodytype.version ./models/bodytype.version
|
||||
RUN --mount=type=secret,id=bodytype_auth \
|
||||
v="$(tr -d '[:space:]' < /app/models/bodytype.version)"; \
|
||||
if [ -n "$v" ]; then \
|
||||
cfg=/tmp/curl.cfg; : > "$cfg"; \
|
||||
[ -f /run/secrets/bodytype_auth ] && printf 'user = "%s"\n' "$(cat /run/secrets/bodytype_auth)" > "$cfg"; \
|
||||
curl -fsSL -K "$cfg" -o /app/models/bodytype.onnx "$BODYTYPE_BASE_URL/$v/bodytype.onnx" \
|
||||
&& curl -fsSL -K "$cfg" -o /app/models/bodytype.json "$BODYTYPE_BASE_URL/$v/bodytype.json" \
|
||||
&& echo "[build] bodytype classifier $v baked" \
|
||||
|| { echo "[build] bodytype classifier $v could not be fetched"; rm -f "$cfg"; exit 1; }; \
|
||||
rm -f "$cfg"; \
|
||||
else echo "[build] no bodytype version pinned — phase B off"; fi
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --extra alpr
|
||||
|
||||
@@ -57,7 +75,8 @@ RUN uv run python -c "from fast_alpr import ALPR; ALPR()" \
|
||||
ENV VISION_RECOGNIZER=stub \
|
||||
VISION_HOST=0.0.0.0 \
|
||||
VISION_PORT=8089 \
|
||||
VISION_VEHICLE_MODEL_PATH=/app/models/yolox_s.onnx
|
||||
VISION_VEHICLE_MODEL_PATH=/app/models/yolox_s.onnx \
|
||||
VISION_VEHICLE_CLASSIFIER_PATH=/app/models/bodytype.onnx
|
||||
EXPOSE 8089
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
||||
CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8089/health').status==200 else 1)" || exit 1
|
||||
|
||||
@@ -143,3 +143,135 @@ def test_app_reports_a_missing_model_file_and_keeps_serving() -> None:
|
||||
assert res.json()["vehicle"] is None
|
||||
finally:
|
||||
os.environ.pop("VISION_VEHICLE_MODEL_PATH", None)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# Phase B: the classifier stage over the detector
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_crop_vehicle_adds_the_margin_clamps_and_blurs_the_plate() -> None:
|
||||
cv2 = pytest.importorskip("cv2")
|
||||
from vision_service.vehicle import crop_vehicle
|
||||
|
||||
frame = np.zeros((100, 200, 3), dtype=np.uint8)
|
||||
frame[40:50, 90:110] = (0, 255, 0) # a green "plate"
|
||||
box = BBox(x1=50, y1=20, x2=150, y2=80) # 100×60 → 8 % margin = 8 / 5 px
|
||||
crop = crop_vehicle(frame, box, None, 0.08)
|
||||
assert crop.shape == (70, 116, 3)
|
||||
edge = crop_vehicle(frame, BBox(x1=0, y1=0, x2=100, y2=60), None, 0.08)
|
||||
assert edge.shape == (65, 108, 3) # clamped at the frame's top-left
|
||||
assert crop_vehicle(frame, BBox(x1=10, y1=10, x2=12, y2=12), None, 0.08) is None
|
||||
blurred = crop_vehicle(frame, box, BBox(x1=90, y1=40, x2=110, y2=50), 0.08)
|
||||
strip = blurred[40 - 20 + 5 : 50 - 20 + 5, 90 - 50 + 8 : 110 - 50 + 8, 1] # plate strip, green channel
|
||||
assert strip.mean() < 200 and crop[20 + 5 : 30 + 5, 40 + 8 : 60 + 8, 1].mean() == 255
|
||||
assert cv2 is not None
|
||||
|
||||
|
||||
class FakeClassifier:
|
||||
ready = True
|
||||
error = None
|
||||
min_confidence = 0.6
|
||||
model_version = "bodytype:vfake"
|
||||
|
||||
def __init__(self, classes: list[str], answer: tuple[str, float] | None) -> None:
|
||||
self.classes = classes
|
||||
self.answer = answer
|
||||
self.calls = 0
|
||||
|
||||
def classify(self, frame, box, plate): # type: ignore[no-untyped-def]
|
||||
self.calls += 1
|
||||
if isinstance(self.answer, Exception):
|
||||
raise self.answer
|
||||
return self.answer
|
||||
|
||||
|
||||
class FrameDetector:
|
||||
"""A detector that answers on decoded frames (like YOLOX) with a fixed result."""
|
||||
|
||||
model_version = "det"
|
||||
ready = True
|
||||
error = None
|
||||
|
||||
def __init__(self, result: VehicleResult | None) -> None:
|
||||
self.result = result
|
||||
|
||||
def detect_frame(self, frame, plate): # type: ignore[no-untyped-def]
|
||||
return self.result
|
||||
|
||||
def detect(self, image_bytes: bytes, plate: BBox | None) -> VehicleResult | None:
|
||||
raise AssertionError("the refined stage should share the decoded frame")
|
||||
|
||||
|
||||
def _jpeg() -> bytes:
|
||||
cv2 = pytest.importorskip("cv2")
|
||||
ok, buf = cv2.imencode(".jpg", np.zeros((60, 80, 3), dtype=np.uint8))
|
||||
assert ok
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def test_refined_detector_replaces_car_when_confident_else_keeps_the_detector() -> None:
|
||||
from vision_service.vehicle import RefinedVehicleDetector
|
||||
|
||||
car = VehicleResult(body_type="car", confidence=0.85, bbox=BBox(x1=10, y1=10, x2=70, y2=50))
|
||||
sure = FakeClassifier(["sedan", "suv"], ("suv", 0.91))
|
||||
res = RefinedVehicleDetector(FrameDetector(car), sure).detect(_jpeg(), None)
|
||||
assert (
|
||||
res is not None and res.body_type == "suv" and res.confidence == 0.91 and res.detector_class == "car"
|
||||
)
|
||||
assert res.bbox == car.bbox
|
||||
|
||||
unsure = FakeClassifier(["sedan", "suv"], ("suv", 0.4))
|
||||
res2 = RefinedVehicleDetector(FrameDetector(car), unsure).detect(_jpeg(), None)
|
||||
assert (
|
||||
res2 is not None
|
||||
and res2.body_type == "car"
|
||||
and res2.confidence == 0.85
|
||||
and res2.detector_class == "car"
|
||||
)
|
||||
|
||||
# A class the classifier never trained on is left alone (its softmax means nothing there).
|
||||
bus = VehicleResult(body_type="bus", confidence=0.9, bbox=car.bbox)
|
||||
skip = FakeClassifier(["sedan", "suv"], ("suv", 0.99))
|
||||
res3 = RefinedVehicleDetector(FrameDetector(bus), skip).detect(_jpeg(), None)
|
||||
assert res3 == bus and skip.calls == 0
|
||||
# …unless it was: a classifier that knows trucks may override a truck.
|
||||
knows = FakeClassifier(["sedan", "truck", "van"], ("van", 0.8))
|
||||
truck = VehicleResult(body_type="truck", confidence=0.7, bbox=car.bbox)
|
||||
res4 = RefinedVehicleDetector(FrameDetector(truck), knows).detect(_jpeg(), None)
|
||||
assert res4 is not None and res4.body_type == "van" and res4.detector_class == "truck"
|
||||
|
||||
|
||||
def test_refined_detector_survives_a_broken_classifier_and_reports_it() -> None:
|
||||
from vision_service.vehicle import RefinedVehicleDetector
|
||||
|
||||
car = VehicleResult(body_type="car", confidence=0.85, bbox=BBox(x1=10, y1=10, x2=70, y2=50))
|
||||
boom = FakeClassifier(["sedan"], RuntimeError("bad graph")) # type: ignore[arg-type]
|
||||
ref = RefinedVehicleDetector(FrameDetector(car), boom)
|
||||
assert ref.detect(_jpeg(), None) == car
|
||||
assert ref.error == "classifier: RuntimeError: bad graph"
|
||||
assert ref.ready is True and ref.model_version == "det+bodytype:vfake"
|
||||
# No box, or a detector that found nothing → nothing to classify.
|
||||
assert RefinedVehicleDetector(FrameDetector(None), boom).detect(_jpeg(), None) is None
|
||||
boxless = VehicleResult(body_type="car", confidence=0.85)
|
||||
assert RefinedVehicleDetector(FrameDetector(boxless), boom).detect(_jpeg(), None) == boxless
|
||||
|
||||
|
||||
def test_classifier_without_files_is_not_ready_and_the_factory_skips_a_missing_model(tmp_path) -> None: # type: ignore[no-untyped-def]
|
||||
from vision_service.recognizer import WithVehicle, build_recognizer
|
||||
from vision_service.settings import Settings
|
||||
from vision_service.vehicle import BodyTypeClassifier, RefinedVehicleDetector
|
||||
|
||||
clf = BodyTypeClassifier(str(tmp_path / "bodytype.onnx"))
|
||||
assert clf.ready is False and "FileNotFoundError" in (clf.error or "")
|
||||
(tmp_path / "bodytype.json").write_text('{"format": "other"}')
|
||||
assert "unknown sidecar format" in (BodyTypeClassifier(str(tmp_path / "bodytype.onnx")).error or "")
|
||||
|
||||
# A path with no file = the normal pre-model state: phase A only, no error in health.
|
||||
s = Settings(
|
||||
vehicle_model_path="/nonexistent/yolox.onnx", vehicle_classifier_path=str(tmp_path / "none.onnx")
|
||||
)
|
||||
rec = build_recognizer(s)
|
||||
assert isinstance(rec, WithVehicle)
|
||||
assert not isinstance(rec._detector, RefinedVehicleDetector) # noqa: SLF001
|
||||
assert "classifier" not in (rec.error or "")
|
||||
|
||||
@@ -14,12 +14,16 @@ Adding a recognizer (e.g. a fine-tuned YOLO + PaddleOCR) = a new class here, no
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
from .schemas import AnalyzeResponse, BBox, PlateResult
|
||||
from .settings import Settings
|
||||
from .vehicle import VehicleDetector, YoloxVehicleDetector
|
||||
from .vehicle import BodyTypeClassifier, RefinedVehicleDetector, VehicleDetector, YoloxVehicleDetector
|
||||
|
||||
log = logging.getLogger("vision")
|
||||
|
||||
|
||||
class Recognizer(Protocol):
|
||||
@@ -215,10 +219,21 @@ def build_recognizer(settings: Settings) -> Recognizer:
|
||||
else:
|
||||
rec = StubRecognizer(settings)
|
||||
if settings.vehicle_model_path:
|
||||
detector = YoloxVehicleDetector(
|
||||
detector: VehicleDetector = YoloxVehicleDetector(
|
||||
settings.vehicle_model_path,
|
||||
input_size=settings.vehicle_input_size,
|
||||
min_confidence=settings.vehicle_min_confidence,
|
||||
)
|
||||
if settings.vehicle_classifier_path:
|
||||
if Path(settings.vehicle_classifier_path).is_file():
|
||||
detector = RefinedVehicleDetector(
|
||||
detector,
|
||||
BodyTypeClassifier(
|
||||
settings.vehicle_classifier_path,
|
||||
min_confidence=settings.vehicle_classifier_min_confidence,
|
||||
),
|
||||
)
|
||||
else:
|
||||
log.info("no body-type classifier at %s — phase B off", settings.vehicle_classifier_path)
|
||||
return WithVehicle(rec, detector)
|
||||
return rec
|
||||
|
||||
@@ -45,6 +45,9 @@ class VehicleResult(BaseModel):
|
||||
confidence: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
# The vehicle's box in frame pixels — the crop a reviewer sees / a classifier eats.
|
||||
bbox: BBox | None = None
|
||||
# Phase B: the detector's coarse class when the body-type classifier ran on this crop
|
||||
# (body_type is then the classifier's answer if confident, else the detector's).
|
||||
detector_class: str | None = None
|
||||
make: str | None = None
|
||||
model: str | None = None
|
||||
|
||||
|
||||
@@ -42,6 +42,14 @@ class Settings(BaseSettings):
|
||||
# site's own, stricter threshold before it FLAGS anything).
|
||||
vehicle_min_confidence: float = 0.4
|
||||
|
||||
# Phase B — the body-type classifier on the detector's crop (bodytype.onnx + its .json
|
||||
# sidecar, produced by apps/trainer, baked into the image when
|
||||
# models/bodytype.version pins a published version). Path set but NO file = the normal
|
||||
# state before the first model ships: the stage is simply off (logged, not an error).
|
||||
vehicle_classifier_path: str | None = None
|
||||
# Below this probability the classifier's answer is dropped and the detector's stands.
|
||||
vehicle_classifier_min_confidence: float = 0.6
|
||||
|
||||
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
|
||||
@@ -229,6 +229,13 @@ class YoloxVehicleDetector:
|
||||
frame = cv2.imdecode(np.frombuffer(image_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
|
||||
if frame is None:
|
||||
return None
|
||||
return self.detect_frame(frame, plate)
|
||||
|
||||
def detect_frame(self, frame: Any, plate: BBox | None) -> VehicleResult | None:
|
||||
"""Same as detect() on an already-decoded BGR frame (the classifier stage decodes
|
||||
once and shares it)."""
|
||||
if self._session is None:
|
||||
return None
|
||||
tensor, scale = letterbox(frame, self._size)
|
||||
raw = self._session.run(None, {self._input_name: tensor})[0][0]
|
||||
found = vehicles_from_output(raw, self._size, scale, self._min_confidence)
|
||||
@@ -242,6 +249,169 @@ class YoloxVehicleDetector:
|
||||
return VehicleResult(body_type=best.body_type, confidence=round(best.confidence, 4), bbox=box)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------------------
|
||||
# Phase B: the body-type classifier on the detector's crop
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
SIDECAR_FORMAT = "parking-bodytype/1"
|
||||
|
||||
|
||||
def crop_vehicle(frame: Any, box: BBox, plate: BBox | None, margin: float) -> Any:
|
||||
"""The detector's box + margin, plate blurred — the SAME cut the collector stores
|
||||
(apps/server review-outbox.ts makeReviewCrop), so the classifier sees at the booth
|
||||
what it was trained on. Returns a BGR array, or None when the box is degenerate."""
|
||||
import cv2
|
||||
|
||||
h, w = frame.shape[:2]
|
||||
mw = round((box.x2 - box.x1) * margin)
|
||||
mh = round((box.y2 - box.y1) * margin)
|
||||
left, top = max(0, box.x1 - mw), max(0, box.y1 - mh)
|
||||
right, bottom = min(w, box.x2 + mw), min(h, box.y2 + mh)
|
||||
if right - left < 8 or bottom - top < 8:
|
||||
return None
|
||||
crop = frame[top:bottom, left:right].copy()
|
||||
if plate is not None:
|
||||
pad = round(max(plate.x2 - plate.x1, plate.y2 - plate.y1) * 0.25)
|
||||
pl, pt = max(0, plate.x1 - pad - left), max(0, plate.y1 - pad - top)
|
||||
pr, pb = min(right - left, plate.x2 + pad - left), min(bottom - top, plate.y2 + pad - top)
|
||||
if pr - pl >= 2 and pb - pt >= 2:
|
||||
sigma = max(6, round((pr - pl) / 6))
|
||||
crop[pt:pb, pl:pr] = cv2.GaussianBlur(crop[pt:pb, pl:pr], (0, 0), sigma)
|
||||
return crop
|
||||
|
||||
|
||||
class BodyTypeClassifier:
|
||||
"""`bodytype.onnx` + its `bodytype.json` sidecar (written by apps/trainer). The sidecar
|
||||
carries the preprocessing contract — class list, input size, crop margin — and the graph
|
||||
normalises internally, so this side only cuts, resizes (INTER_AREA, like the trainer)
|
||||
and feeds raw RGB 0–255. Load failure → `error`, the stage yields nothing."""
|
||||
|
||||
def __init__(self, model_path: str, min_confidence: float = 0.6) -> None:
|
||||
import json
|
||||
|
||||
self._path = Path(model_path)
|
||||
self.min_confidence = min_confidence
|
||||
self._session = None
|
||||
self._input_name = "image"
|
||||
self._error: str | None = None
|
||||
self.classes: list[str] = []
|
||||
self.version = "?"
|
||||
self.input_size = 224
|
||||
self.crop_margin = 0.08
|
||||
try:
|
||||
side = json.loads(self._path.with_suffix(".json").read_text())
|
||||
if side.get("format") != SIDECAR_FORMAT:
|
||||
raise ValueError(f"unknown sidecar format {side.get('format')!r}")
|
||||
self.classes = [str(c) for c in side["classes"]]
|
||||
self.version = str(side.get("version", "?"))
|
||||
self.input_size = int(side.get("input_size", 224))
|
||||
self.crop_margin = float(side.get("crop_margin", 0.08))
|
||||
import onnxruntime as ort
|
||||
|
||||
opts = ort.SessionOptions()
|
||||
opts.intra_op_num_threads = 2
|
||||
self._session = ort.InferenceSession(
|
||||
str(self._path), sess_options=opts, providers=["CPUExecutionProvider"]
|
||||
)
|
||||
self._input_name = self._session.get_inputs()[0].name
|
||||
except Exception as exc: # noqa: BLE001 - not-ready, never fatal
|
||||
self._error = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
@property
|
||||
def model_version(self) -> str:
|
||||
return f"bodytype:{self.version}"
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self._session is not None
|
||||
|
||||
@property
|
||||
def error(self) -> str | None:
|
||||
return self._error
|
||||
|
||||
def classify(self, frame: Any, box: BBox, plate: BBox | None) -> tuple[str, float] | None:
|
||||
"""(class, probability) for the vehicle in `box`, or None when nothing could be cut."""
|
||||
if self._session is None:
|
||||
return None
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
crop = crop_vehicle(frame, box, plate, self.crop_margin)
|
||||
if crop is None:
|
||||
return None
|
||||
resized = cv2.resize(crop, (self.input_size, self.input_size), interpolation=cv2.INTER_AREA)
|
||||
rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)
|
||||
x = np.ascontiguousarray(rgb.transpose(2, 0, 1)[None].astype(np.float32))
|
||||
logits = self._session.run(None, {self._input_name: x})[0][0]
|
||||
z = logits - logits.max()
|
||||
p = np.exp(z) / np.exp(z).sum()
|
||||
i = int(p.argmax())
|
||||
return self.classes[i], float(p[i])
|
||||
|
||||
|
||||
class RefinedVehicleDetector:
|
||||
"""Detector + classifier. The detector finds the vehicle (and picks WHICH one); when its
|
||||
class is `car` — or one the classifier was trained on — the classifier's answer replaces
|
||||
it if confident enough, else the detector's stands. A truck or bus the classifier has
|
||||
never seen is left alone: its softmax on an unknown thing means nothing."""
|
||||
|
||||
def __init__(self, detector: Any, classifier: BodyTypeClassifier) -> None:
|
||||
self._detector = detector
|
||||
self._classifier = classifier
|
||||
self.stage_error: str | None = None
|
||||
|
||||
@property
|
||||
def model_version(self) -> str:
|
||||
return f"{self._detector.model_version}+{self._classifier.model_version}"
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return bool(getattr(self._detector, "ready", True))
|
||||
|
||||
@property
|
||||
def error(self) -> str | None:
|
||||
parts = [
|
||||
getattr(self._detector, "error", None),
|
||||
f"classifier: {self._classifier.error}" if self._classifier.error else None,
|
||||
f"classifier: {self.stage_error}" if self.stage_error else None,
|
||||
]
|
||||
kept = [p for p in parts if p]
|
||||
return "; ".join(kept) if kept else None
|
||||
|
||||
def detect(self, image_bytes: bytes, plate: BBox | None) -> VehicleResult | None:
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
frame = cv2.imdecode(np.frombuffer(image_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
|
||||
if frame is None:
|
||||
return None
|
||||
detect_frame = getattr(self._detector, "detect_frame", None)
|
||||
base: VehicleResult | None = (
|
||||
detect_frame(frame, plate) if detect_frame else self._detector.detect(image_bytes, plate)
|
||||
)
|
||||
if base is None or base.bbox is None or not self._classifier.ready:
|
||||
return base
|
||||
if not (base.body_type == "car" or base.body_type in self._classifier.classes):
|
||||
return base
|
||||
try:
|
||||
out = self._classifier.classify(frame, base.bbox, plate)
|
||||
except Exception as exc: # noqa: BLE001 - advisory stage, never fatal
|
||||
self.stage_error = f"{type(exc).__name__}: {exc}"
|
||||
return base
|
||||
if out is None:
|
||||
return base
|
||||
body_type, confidence = out
|
||||
if confidence < self._classifier.min_confidence:
|
||||
return base.model_copy(update={"detector_class": base.body_type})
|
||||
return base.model_copy(
|
||||
update={
|
||||
"body_type": body_type,
|
||||
"confidence": round(confidence, 4),
|
||||
"detector_class": base.body_type,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def time_detect(
|
||||
detector: VehicleDetector, image_bytes: bytes, plate: BBox | None
|
||||
) -> tuple[VehicleResult | None, float]:
|
||||
|
||||
@@ -22,26 +22,30 @@ services:
|
||||
COLLECTOR_REVIEWER_USER: ${COLLECTOR_REVIEWER_USER:-reviewer}
|
||||
COLLECTOR_REVIEWER_PASS: ${COLLECTOR_REVIEWER_PASS:?set COLLECTOR_REVIEWER_PASS in the stack env}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
# The trainer's job API (sibling service above). Unset = no Training section.
|
||||
COLLECTOR_TRAINER_URL: ${COLLECTOR_TRAINER_URL-http://trainer:8091}
|
||||
volumes:
|
||||
- collector-data:/data
|
||||
|
||||
# Phase B trainer — a one-off job on this host's GPU, NOT a service (profile "train": it
|
||||
# only runs when asked: `docker compose --profile train run --rm trainer`). Reads the
|
||||
# collector's export + crops straight off the same volume; writes the ONNX classifier the
|
||||
# vision image then bakes in. The image/script are the next increment; this is the seam.
|
||||
# trainer:
|
||||
# image: ${REGISTRY:-git.infra.msai.al/mca/parking_solution}/parking-trainer:${TAG:-dev}
|
||||
# profiles: ["train"]
|
||||
# deploy:
|
||||
# resources:
|
||||
# reservations:
|
||||
# devices:
|
||||
# - driver: nvidia
|
||||
# count: all
|
||||
# capabilities: [gpu]
|
||||
# volumes:
|
||||
# - collector-data:/data:ro
|
||||
# - ./models:/out
|
||||
# Phase B trainer — a small always-on job service beside the collector (CPU-only torch;
|
||||
# idle it is a tiny Python HTTP server, torch loads only when a job runs). It reads the
|
||||
# collector's SQLite + crops off the same volume (read-only) and keeps models, reports and
|
||||
# job logs in its own volume. NOT published: only the collector reaches it, on this compose
|
||||
# network, and the reviewer's login on the collector is the gate. The Training section of
|
||||
# /review is its UI (readiness, Train / Evaluate / Publish, reports, logs).
|
||||
# See wiki/decisions/bodytype-classifier-training.md.
|
||||
trainer:
|
||||
image: ${REGISTRY:-git.infra.msai.al/mca/parking_solution}/parking-trainer:${TAG:-dev}
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# Where `publish` PUTs a passing model (a Gitea generic package) and the token it uses
|
||||
# (package:write). Only publishing needs the token; training runs without it.
|
||||
TRAINER_PUBLISH_URL: ${TRAINER_PUBLISH_URL:-https://git.infra.msai.al/api/packages/mca/generic/parking-bodytype}
|
||||
TRAINER_PUBLISH_TOKEN: ${TRAINER_PUBLISH_TOKEN:-}
|
||||
volumes:
|
||||
- collector-data:/data:ro
|
||||
- trainer-out:/out
|
||||
|
||||
volumes:
|
||||
collector-data:
|
||||
trainer-out:
|
||||
|
||||
+64
-4
@@ -49,11 +49,20 @@ REGISTRY=git.infra.msai.al/mca/parking_solution
|
||||
# Staging booth: pinned immutable stage-<sha>. After each promotion (merge dev → stage, CI builds
|
||||
# :stage-<sha>), bump this to the new sha and re-sync/deploy from Core. The moving `:stage` tag
|
||||
# exists as the pointer; we deploy the sha, not the mover.
|
||||
TAG=stage-8fa66c9
|
||||
TAG=stage-88f9c53
|
||||
COOKIE_SECURE=0
|
||||
# Venue modules this site is ENTITLED to (vendor decision; the site admin activates within
|
||||
# this set in Setup → Site). Unset = every registered module. See wiki/decisions/venue-modules.md.
|
||||
MODULES_ENTITLED=parking,validation
|
||||
# Car Wash review outbox (wiki/concepts/vision-review-outbox.md): the collector's ingest URL
|
||||
# on the Netbird overlay, this booth's PSEUDONYMOUS id (never the site name — the crops leave
|
||||
# the site), and its token — the SAME secret the wash-collector stack lists under that id.
|
||||
CARWASH_REVIEW_URL=http://docker-station.nb.infra:8090/ingest
|
||||
CARWASH_REVIEW_BOOTH_ID=booth-1
|
||||
CARWASH_REVIEW_TOKEN=[[wash_review_token_booth_buzi]]
|
||||
# Also send ENTRY reads as training material (gate view, no wash): 1 = every entry (storage
|
||||
# and bandwidth are not the limit; review what you have time for). N = one in N. 0 = off.
|
||||
CARWASH_REVIEW_ENTRY_SAMPLE=1
|
||||
VISION_ENABLED=1
|
||||
# Desktop app WS handshake: Origin is tauri://localhost (set explicitly by
|
||||
# platform-ws.ts, since the native WS plugin has no page context to auto-attach
|
||||
@@ -85,7 +94,7 @@ REGISTRY=git.infra.msai.al/mca/parking_solution
|
||||
# Staging booth: pinned immutable stage-<sha>. After each promotion (merge dev → stage, CI builds
|
||||
# :stage-<sha>), bump this to the new sha and re-sync/deploy from Core. The moving `:stage` tag
|
||||
# exists as the pointer; we deploy the sha, not the mover.
|
||||
TAG=stage-dbbb051
|
||||
TAG=stage-88f9c53
|
||||
COOKIE_SECURE=0
|
||||
# Venue modules this site is ENTITLED to (vendor decision; the site admin activates within
|
||||
# this set in Setup → Site). Unset = every registered module. See wiki/decisions/venue-modules.md.
|
||||
@@ -110,6 +119,53 @@ EVENT_SIGNING_KEY=[[park_2_event_signing_key]]
|
||||
BACKUP_KEY=[[park_2_backup_key]]
|
||||
"""
|
||||
|
||||
##############################################################################
|
||||
# Stack — the LAB BENCH (not a booth): a spare Linux box with the field printer and
|
||||
# whatever device is under investigation, so a booth bug can be reproduced on the booth's
|
||||
# exact image before touching a real site. Same compose files + pinned TAG as the staging
|
||||
# booths. Review outbox under its OWN id (booth-lab) — never a
|
||||
# booth's identity, so bench crops stay separable on the collector. Its own secrets. See wiki/decisions/fleet-deployment-komodo.md.
|
||||
##############################################################################
|
||||
|
||||
[[stack]]
|
||||
name = "park-lab"
|
||||
[stack.config]
|
||||
server = "park-lab"
|
||||
git_provider = "git.infra.msai.al"
|
||||
git_account = "komodo"
|
||||
repo = "mca/parking_solution"
|
||||
branch = "stage"
|
||||
file_paths = [
|
||||
"docker-compose.yml",
|
||||
"docker-compose.prod.yml"
|
||||
]
|
||||
registry_provider = "git.infra.msai.al"
|
||||
registry_account = "komodo"
|
||||
environment = """
|
||||
REGISTRY=git.infra.msai.al/mca/parking_solution
|
||||
# Lab: pinned to the SAME stage-<sha> as the booth whose bug is being reproduced (bump
|
||||
# alongside it). A lab may float, but a reproduction must run the booth's exact image.
|
||||
TAG=stage-88f9c53
|
||||
COOKIE_SECURE=0
|
||||
# Entitled to Car Wash too, so the wash-desk printer role and till can be exercised on the bench.
|
||||
MODULES_ENTITLED=parking,carwash
|
||||
# Review outbox ON for the bench too, under its OWN pseudonymous id (booth-lab) and its own
|
||||
# secret — never a real booth's — so lab crops are separable on the collector (per-booth stats,
|
||||
# the export's booth column) and the reviewer can label bench junk "unusable".
|
||||
CARWASH_REVIEW_URL=http://docker-station.nb.infra:8090/ingest
|
||||
CARWASH_REVIEW_BOOTH_ID=booth-lab
|
||||
CARWASH_REVIEW_TOKEN=[[wash_review_token_booth_lab]]
|
||||
CARWASH_REVIEW_ENTRY_SAMPLE=1
|
||||
VISION_ENABLED=1
|
||||
# Desktop app WS handshake: Origin is tauri://localhost (set explicitly by
|
||||
# platform-ws.ts, since the native WS plugin has no page context to auto-attach
|
||||
# one). Linux may also send http://tauri.localhost. See routes/ws.ts anti-CSWSH check.
|
||||
WS_ALLOWED_ORIGINS=tauri://localhost,http://tauri.localhost
|
||||
JWT_SECRET=[[park_lab_jwt_secret]]
|
||||
EVENT_SIGNING_KEY=[[park_lab_event_signing_key]]
|
||||
BACKUP_KEY=[[park_lab_backup_key]]
|
||||
"""
|
||||
|
||||
##############################################################################
|
||||
# Stack — the Car Wash REVIEW COLLECTOR on the reviewer's host (art-docker-station),
|
||||
# NOT a booth. Same repo/branch/TAG promotion as the booths, but its file_paths name
|
||||
@@ -134,7 +190,7 @@ registry_account = "komodo"
|
||||
environment = """
|
||||
REGISTRY=git.infra.msai.al/mca/parking_solution
|
||||
# Pinned like the booths: bump to the stage-<sha> that carries the collector.
|
||||
TAG=stage-dbbb051
|
||||
TAG=stage-73bfc1d
|
||||
# The host's NETBIRD address (an IP: Docker port bindings take no hostname) — the ingest port
|
||||
# is published on the overlay only. Booths reach it by its Netbird DNS name.
|
||||
COLLECTOR_BIND=100.75.184.156
|
||||
@@ -142,7 +198,11 @@ COLLECTOR_BIND=100.75.184.156
|
||||
# that booth's own stack as its CARWASH_REVIEW_TOKEN — one value, two consumers, nothing
|
||||
# to keep in sync, and rotating a booth touches one secret. The booth id is the booth's
|
||||
# pseudonymous CARWASH_REVIEW_BOOTH_ID, never a site name. Add a pair per booth.
|
||||
COLLECTOR_BOOTH_TOKENS=booth-2:[[wash_review_token_booth_2]]
|
||||
COLLECTOR_BOOTH_TOKENS=booth-2:[[wash_review_token_booth_2]],booth-1:[[wash_review_token_booth_buzi]],booth-lab:[[wash_review_token_booth_lab]]
|
||||
# Phase-B trainer (the `trainer` service beside the collector; the Training section of /review
|
||||
# is its UI). Only `publish` needs this: a Gitea token with package:write for the model's generic
|
||||
# package. Uncomment when the first model is to be published.
|
||||
#TRAINER_PUBLISH_TOKEN=[[gitea_package_write_token]]
|
||||
COLLECTOR_REVIEWER_USER=reviewer
|
||||
COLLECTOR_REVIEWER_PASS=[[wash_collector_reviewer_pass]]
|
||||
"""
|
||||
|
||||
@@ -51,6 +51,10 @@ const CATEGORIES = {
|
||||
"subscription_credentials",
|
||||
"subscriptions",
|
||||
"blocklist",
|
||||
// Car Wash (venue module): orders are money history (settled against ledger events);
|
||||
// the review outbox is a delivery queue of crops + choices — both go with the ledger.
|
||||
"carwash_review_outbox",
|
||||
"carwash_orders",
|
||||
],
|
||||
config: [
|
||||
"site_config",
|
||||
@@ -65,8 +69,15 @@ const CATEGORIES = {
|
||||
// whose user is gone grants nothing.
|
||||
"validation_program_users",
|
||||
"validation_programs",
|
||||
// Car Wash master data: prices reference categories + services (child first); the
|
||||
// module's site-level config (pay-at, vision threshold) is config like site_config.
|
||||
"carwash_prices",
|
||||
"carwash_categories",
|
||||
"carwash_services",
|
||||
"carwash_config",
|
||||
],
|
||||
users: ["sessions", "role_permissions", "users", "roles"],
|
||||
// role_jobs = which jobs a role follows (child of roles).
|
||||
users: ["sessions", "role_permissions", "role_jobs", "users", "roles"],
|
||||
diagnostics: ["app_logs"],
|
||||
};
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { dingtianDriver } from "./access-dingtian.js";
|
||||
import { stubAccessDriver } from "./access-stub.js";
|
||||
import { dahuaDriver, hikvisionDriver } from "./camera.js";
|
||||
import { escposDriver } from "./printer-generic.js";
|
||||
import { k200lDriver } from "./printer-k200l.js";
|
||||
import { rongtaDriver } from "./printer-rongta.js";
|
||||
import { dingtianQrReaderDriver, tcpipReaderDriver, wiegandReaderDriver } from "./reader.js";
|
||||
|
||||
@@ -23,6 +24,7 @@ export function registerBuiltinDrivers(): void {
|
||||
registry.register(hikvisionDriver);
|
||||
registry.register(dahuaDriver);
|
||||
registry.register(rongtaDriver);
|
||||
registry.register(k200lDriver);
|
||||
registry.register(escposDriver);
|
||||
}
|
||||
|
||||
@@ -35,5 +37,6 @@ export {
|
||||
hikvisionDriver,
|
||||
dahuaDriver,
|
||||
rongtaDriver,
|
||||
k200lDriver,
|
||||
escposDriver,
|
||||
};
|
||||
|
||||
@@ -37,6 +37,8 @@ import {
|
||||
// local usblp char device (/dev/usb/lp0); TCP writes to the raw print socket. This
|
||||
// clone family is the natural USB candidate — reachability-only, no page to lose.
|
||||
//
|
||||
// (The K200L / XP-K200L is the exception: its LAN board DOES serve a status page,
|
||||
// /prt_status.htm — use the `k200l` driver for it over TCP; see printer-k200l.ts.)
|
||||
// Therefore this driver deliberately does NOT implement MonitorableDevice
|
||||
// (no readStatus). The device monitor then falls back to the generic
|
||||
// `healthCheck()` — a plain TCP reachability PING of the print socket. So the
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createServer, type Server } from "node:net";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { parseRawReply, parseStatusPage, k200lDriver } from "./printer-k200l.js";
|
||||
import { renderTicket } from "./printer-escpos.js";
|
||||
import type { MonitorableDevice, PrinterDevice } from "../interfaces.js";
|
||||
|
||||
// The K200L (Xprinter / ICS; J-Speed 'POS-80' LAN board) driver. Its status page was captured verbatim
|
||||
// from the unit on the lab bench, 2026-09-09: uppercase tags, values padded with
|
||||
// spaces, and — the part that matters — the board's reply has NO status line and NO
|
||||
// headers (the body starts at byte 0). The tests replay exactly that over a raw
|
||||
// socket, plus a proper-HTTP variant, so the driver is proven against both.
|
||||
|
||||
/** The board's status table, as sent (CRLF, uppercase, padded values). */
|
||||
function boardPage(flags: Partial<Record<string, "Yes" | "No">> = {}): string {
|
||||
const v = (k: string) => `${flags[k] ?? "No"} `;
|
||||
return [
|
||||
'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">',
|
||||
"<HTML><HEAD><TITLE>Printer Status</TITLE>",
|
||||
"<META http-equiv=refresh content=\"5;url='prt_status.htm'\"></HEAD>",
|
||||
'<BODY><FORM id=Form1 action="prt_status.htm" method="get">',
|
||||
"<TABLE id=Table3 cellPadding=3 border=0><TBODY>",
|
||||
`<TR><TD>Cover Is Open</TD><TD style="width: 23px">${v("cover")}</TD></TR>`,
|
||||
`<TR><TD>Cutter Error</TD><TD style="width: 23px">${v("cutter")}</TD></TR>`,
|
||||
`<TR><TD>Paper End</TD><TD style="width: 23px">${v("paperEnd")}</TD></TR>`,
|
||||
`<TR><TD>Paper Near End</TD><TD style="width: 23px">${v("nearEnd")}</TD></TR>`,
|
||||
`<TR><TD>Printer Off-Line</TD><TD style="width: 23px">${v("offline")}</TD></TR></TBODY></TABLE>`,
|
||||
'<INPUT type=submit value="Print Test Page" name=page_p2></FORM></BODY></HTML>',
|
||||
].join("\r\n");
|
||||
}
|
||||
|
||||
const INDEX =
|
||||
"<HTML><HEAD><TITLE>Ethernet port configuration</TITLE></HEAD><BODY><TABLE><TR><TD>Mac Address</TD><TD>00-D8-23-5C-58-8C</TD></TR></TABLE></BODY></HTML>";
|
||||
|
||||
type Reply = { body: string; status?: number; raw?: boolean };
|
||||
|
||||
describe("parseRawReply", () => {
|
||||
it("treats a reply without a status line as HTTP/0.9: the whole reply is the body", () => {
|
||||
const r = parseRawReply("<!DOCTYPE HTML><HTML>x</HTML>");
|
||||
expect(r.status).toBe(200);
|
||||
expect(r.body).toBe("<!DOCTYPE HTML><HTML>x</HTML>");
|
||||
});
|
||||
it("splits a real HTTP reply into status and body", () => {
|
||||
const r = parseRawReply("HTTP/1.0 404 Not Found\r\nContent-Type: text/html\r\n\r\n<b>nope</b>");
|
||||
expect(r.status).toBe(404);
|
||||
expect(r.body).toBe("<b>nope</b>");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseStatusPage", () => {
|
||||
it("reads the board's padded, uppercase table", () => {
|
||||
const f = parseStatusPage(boardPage({ cover: "Yes", nearEnd: "Yes" }));
|
||||
expect(f).toEqual({ coverOpen: true, cutterError: false, paperEnd: false, paperNearEnd: true, offline: false });
|
||||
});
|
||||
it("leaves unknown pages empty rather than guessing", () => {
|
||||
expect(parseStatusPage(INDEX)).toEqual({});
|
||||
});
|
||||
it("reads a fault's Yes, which the board wraps in <FONT color=#ff0000> (captured live, cover open)", () => {
|
||||
// Verbatim from the unit on 2026-09-09 with the cover open: the three fault cells carry
|
||||
// markup the No cells don't — the first parser rejected them ("missing coverOpen,
|
||||
// paperEnd, offline" on the booth) while the No cells parsed.
|
||||
const page = boardPage()
|
||||
.replace("Cover Is Open</TD><TD style=\"width: 23px\">No ", "Cover Is Open</TD><TD style=\"width: 23px\"><FONT color=#ff0000>Yes</FONT> ")
|
||||
.replace("Paper End</TD><TD style=\"width: 23px\">No ", "Paper End</TD><TD style=\"width: 23px\"><FONT color=#ff0000>Yes</FONT> ")
|
||||
.replace("Printer Off-Line</TD><TD style=\"width: 23px\">No ", "Printer Off-Line</TD><TD style=\"width: 23px\"><FONT color=#ff0000>Yes</FONT> ");
|
||||
expect(parseStatusPage(page)).toEqual({ coverOpen: true, cutterError: false, paperEnd: true, paperNearEnd: false, offline: true });
|
||||
});
|
||||
it("tolerates other markup shapes around a value", () => {
|
||||
const page = boardPage()
|
||||
.replace("Paper End</TD><TD style=\"width: 23px\">No ", "Paper End</TD><TD style=\"width: 23px\"><B><FONT color=\"#ff0000\">Yes </FONT></B>")
|
||||
.replace("Printer Off-Line</TD><TD style=\"width: 23px\">No ", "Printer Off-Line</TD>\r\n<TD style=\"width: 23px\">\r\n<font>Yes</font>\r\n");
|
||||
expect(parseStatusPage(page)).toEqual({ coverOpen: false, cutterError: false, paperEnd: true, paperNearEnd: false, offline: true });
|
||||
});
|
||||
it("reads labels wrapped in markup too", () => {
|
||||
const page = boardPage({ nearEnd: "Yes" }).replace("<TD>Paper Near End</TD>", "<TD><B>Paper Near End</B></TD>");
|
||||
expect(parseStatusPage(page).paperNearEnd).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("k200lDriver.readStatus over TCP", () => {
|
||||
let server: Server | undefined;
|
||||
const sockets = new Set<import("node:net").Socket>();
|
||||
|
||||
/** A raw TCP server that answers like the board (no status line) unless the reply
|
||||
* says otherwise, and closes after the reply (HTTP/1.0). */
|
||||
async function serve(reply: (path: string) => Reply): Promise<number> {
|
||||
server = createServer((sock) => {
|
||||
sockets.add(sock);
|
||||
sock.on("close", () => sockets.delete(sock));
|
||||
sock.once("data", (d) => {
|
||||
const path = /^GET (\S+)/.exec(d.toString())?.[1] ?? "";
|
||||
const r = reply(path);
|
||||
if (r.raw === false) {
|
||||
sock.end(`HTTP/1.0 ${r.status ?? 200} OK\r\nContent-Type: text/html\r\n\r\n${r.body}`);
|
||||
} else {
|
||||
sock.end(r.body);
|
||||
}
|
||||
});
|
||||
});
|
||||
await new Promise<void>((r) => server!.listen(0, "127.0.0.1", () => r()));
|
||||
const addr = server.address();
|
||||
if (!addr || typeof addr === "string") throw new Error("no port");
|
||||
return addr.port;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const s of sockets) s.destroy();
|
||||
sockets.clear();
|
||||
if (server) await new Promise<void>((r) => server!.close(() => r()));
|
||||
server = undefined;
|
||||
});
|
||||
|
||||
function device(httpPort: number): MonitorableDevice & { driverId: string } {
|
||||
return k200lDriver.create({ transport: "tcp-ip", host: "127.0.0.1", port: 9100, httpPort, timeoutMs: 1000 }) as unknown as MonitorableDevice & {
|
||||
driverId: string;
|
||||
};
|
||||
}
|
||||
|
||||
it("reads the headerless reply: cover open + paper out + off-line → degraded, flags set", async () => {
|
||||
const port = await serve((p) => (p === "/prt_status.htm" ? { body: boardPage({ cover: "Yes", paperEnd: "Yes", offline: "Yes" }) } : { body: INDEX }));
|
||||
const dev = device(port);
|
||||
expect(dev.driverId).toBe("k200l");
|
||||
const s = await dev.readStatus();
|
||||
expect(s.status).toBe("degraded");
|
||||
expect(s.coverOpen).toBe(true);
|
||||
expect(s.paperEnd).toBe(true);
|
||||
expect(s.offline).toBe(true);
|
||||
expect(s.cutterError).toBe(false);
|
||||
expect(s.paperNearEnd).toBe(false);
|
||||
expect(s.detail).toBe("cover open, paper out, printer off-line");
|
||||
});
|
||||
|
||||
it("healthy printer → ready, every flag false", async () => {
|
||||
const port = await serve(() => ({ body: boardPage() }));
|
||||
const s = await device(port).readStatus();
|
||||
expect(s.status).toBe("ready");
|
||||
expect(s.coverOpen).toBe(false);
|
||||
expect(s.detail).toBeUndefined();
|
||||
});
|
||||
|
||||
it("paper near end alone → degraded 'paper low' (still prints, warn to reload)", async () => {
|
||||
const port = await serve(() => ({ body: boardPage({ nearEnd: "Yes" }) }));
|
||||
const s = await device(port).readStatus();
|
||||
expect(s.status).toBe("degraded");
|
||||
expect(s.paperNearEnd).toBe(true);
|
||||
expect(s.detail).toBe("paper low");
|
||||
});
|
||||
|
||||
it("also understands a proper HTTP reply (a board firmware that sends headers)", async () => {
|
||||
const port = await serve(() => ({ body: boardPage({ cutter: "Yes" }), raw: false }));
|
||||
const s = await device(port).readStatus();
|
||||
expect(s.status).toBe("degraded");
|
||||
expect(s.detail).toBe("cutter error");
|
||||
});
|
||||
|
||||
it("a page without the status rows (the index) → degraded 'unexpected status page', never ready", async () => {
|
||||
const port = await serve(() => ({ body: INDEX }));
|
||||
const s = await device(port).readStatus();
|
||||
expect(s.status).toBe("degraded");
|
||||
expect(s.detail).toContain("unexpected status page");
|
||||
expect(s.detail).toContain("missing");
|
||||
});
|
||||
|
||||
it("a non-200 reply → degraded naming the code, never ready", async () => {
|
||||
const port = await serve(() => ({ body: "", status: 404, raw: false }));
|
||||
const s = await device(port).readStatus();
|
||||
expect(s.status).toBe("degraded");
|
||||
expect(s.detail).toContain("HTTP 404");
|
||||
});
|
||||
|
||||
it("board unreachable (connection refused) → offline", async () => {
|
||||
const port = await serve(() => ({ body: "" }));
|
||||
await new Promise<void>((r) => server!.close(() => r()));
|
||||
server = undefined;
|
||||
const s = await device(port).readStatus();
|
||||
expect(s.status).toBe("offline");
|
||||
expect(s.detail).toMatch(/ECONNREFUSED/);
|
||||
});
|
||||
|
||||
it("a board that accepts but never answers → offline 'status page timeout'", async () => {
|
||||
server = createServer((sock) => {
|
||||
sockets.add(sock); // hold the socket open, say nothing
|
||||
sock.on("close", () => sockets.delete(sock));
|
||||
});
|
||||
await new Promise<void>((r) => server!.listen(0, "127.0.0.1", () => r()));
|
||||
const addr = server.address();
|
||||
if (!addr || typeof addr === "string") throw new Error("no port");
|
||||
const dev = k200lDriver.create({ transport: "tcp-ip", host: "127.0.0.1", port: 9100, httpPort: addr.port, timeoutMs: 200 }) as unknown as MonitorableDevice;
|
||||
const s = await dev.readStatus();
|
||||
expect(s.status).toBe("offline");
|
||||
expect(s.detail).toBe("status page timeout");
|
||||
});
|
||||
});
|
||||
|
||||
describe("k200lDriver — printing and USB are the generic ESC/POS path", () => {
|
||||
let dir: string;
|
||||
let devicePath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), "k200l-usb-"));
|
||||
devicePath = join(dir, "lp0");
|
||||
writeFileSync(devicePath, "");
|
||||
});
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("prints the same ticket bytes the generic driver would, to the USB node", async () => {
|
||||
const printer = k200lDriver.create({ transport: "usb", devicePath, timeoutMs: 1000 }) as PrinterDevice;
|
||||
const data = { ticketId: "12345678901", issuedAt: "2026-09-09T10:00:00.000Z" };
|
||||
await printer.printTicket(data);
|
||||
expect(readFileSync(devicePath).equals(renderTicket(data))).toBe(true);
|
||||
});
|
||||
|
||||
it("over USB readStatus is the reachability floor: ready when the node opens, offline when absent", async () => {
|
||||
const present = k200lDriver.create({ transport: "usb", devicePath, timeoutMs: 1000 }) as unknown as MonitorableDevice;
|
||||
expect((await present.readStatus()).status).toBe("ready");
|
||||
const absent = k200lDriver.create({ transport: "usb", devicePath: join(dir, "absent"), timeoutMs: 1000 }) as unknown as MonitorableDevice;
|
||||
expect((await absent.readStatus()).status).toBe("offline");
|
||||
});
|
||||
|
||||
it("advertises both transports and exposes the status-page port after the print port", () => {
|
||||
expect(k200lDriver.transports).toEqual(["tcp-ip", "usb"]);
|
||||
const keys = k200lDriver.configFields.map((f) => f.key);
|
||||
expect(keys.indexOf("httpPort")).toBe(keys.indexOf("port") + 1);
|
||||
expect(keys).toContain("role");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,262 @@
|
||||
import { connect as netConnect } from "node:net";
|
||||
import type {
|
||||
DeviceHealth,
|
||||
MonitorableDevice,
|
||||
PrinterDevice,
|
||||
PrinterStatus,
|
||||
PrintReport,
|
||||
ReceiptData,
|
||||
SubscriptionCardData,
|
||||
TicketData,
|
||||
WindowChargeNoticeData,
|
||||
} from "../interfaces.js";
|
||||
import type { ConfigField, DeviceConfig, PrinterDriver } from "../registry.js";
|
||||
import { stubLog } from "./common.js";
|
||||
import { transportFromConfig } from "./printer-escpos.js";
|
||||
import { escposDriver } from "./printer-generic.js";
|
||||
|
||||
// K200L 80mm ESC/POS thermal printer (Xprinter / ICS "XP-K200L" family; label:
|
||||
// "THERMAL RECEIPT PRINTER Model:K200L, Interface: USB+LAN, Command Support: ESC/POS").
|
||||
// Its LAN board is the "J-Speed Ethernet Interface Module" (web UI "Ethernet WebConfig
|
||||
// 1.02") and calls the printer "POS-80" — over USB it enumerates as 1fc9:2016
|
||||
// "Printer POS-80". Identified on the lab bench 2026-09-09: it is the park-buzi unit.
|
||||
// See wiki/entities/k200l-printer.md.
|
||||
//
|
||||
// PRINTING is the shared ESC/POS path (delegated to the generic driver — same bytes,
|
||||
// same TCP-9100 / usblp transports). What this driver ADDS is live status: the board
|
||||
// serves a status page, /prt_status.htm, with the same five decoded Yes/No rows the
|
||||
// Rongta board serves under /prn_stat.htm (cover open, cutter error, paper end, paper
|
||||
// near end, off-line). So over TCP the operator gets a real paper/cover verdict —
|
||||
// the generic driver deliberately can't (reachability only), and the Rongta driver
|
||||
// can't read THIS board either: its reply carries NO status line and NO headers
|
||||
// (HTTP/0.9 style — the body starts at byte 0), which Node's http client rejects
|
||||
// ("Parse Error: Expected HTTP/"). Hence the raw-socket fetch below, tolerant of both
|
||||
// shapes. Over USB there is no page; status degrades to the reachability floor.
|
||||
//
|
||||
// Board facts worth knowing (all verified on the bench): factory address
|
||||
// 192.168.123.100, DHCP off; web configurator on port 80 (Information / Configuration
|
||||
// / Printer Status / Printer Test); the frameset reloads its frames every 1–3 s and
|
||||
// the status page every 5 s, and the embedded HTTP server is tiny — leave the browser
|
||||
// closed while the monitor polls, or connects will intermittently time out.
|
||||
|
||||
/** The fault flags the status page reports (a subset of PrinterStatus). */
|
||||
type StatusFlag = "coverOpen" | "cutterError" | "paperEnd" | "paperNearEnd" | "offline";
|
||||
type StatusFlags = Partial<Record<StatusFlag, boolean>>;
|
||||
|
||||
/** Label text on the status page (space-normalised, lowercased) → our key. */
|
||||
const STATUS_FIELDS: Record<string, StatusFlag> = {
|
||||
"cover is open": "coverOpen",
|
||||
"cutter error": "cutterError",
|
||||
"paper end": "paperEnd",
|
||||
"paper near end": "paperNearEnd",
|
||||
"printer off-line": "offline",
|
||||
};
|
||||
const EXPECTED: readonly StatusFlag[] = ["coverOpen", "cutterError", "paperEnd", "paperNearEnd", "offline"];
|
||||
const LABELS: Record<StatusFlag, string> = {
|
||||
paperEnd: "paper out",
|
||||
coverOpen: "cover open",
|
||||
cutterError: "cutter error",
|
||||
offline: "printer off-line",
|
||||
paperNearEnd: "paper low",
|
||||
};
|
||||
|
||||
/** The board's status page. */
|
||||
export const K200L_STATUS_PATH = "/prt_status.htm";
|
||||
|
||||
/**
|
||||
* GET `path` over a raw TCP socket and return whatever the board sent, verbatim,
|
||||
* once it closes the connection (HTTP/1.0 semantics — the board closes after the
|
||||
* reply). No HTTP parsing here: this board answers without a status line, which
|
||||
* node:http refuses to parse.
|
||||
*/
|
||||
export function fetchRaw(host: string, port: number, path: string, timeoutMs: number): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let settled = false;
|
||||
const sock = netConnect({ host, port });
|
||||
const timer = setTimeout(() => {
|
||||
finish(() => reject(new Error("status page timeout")));
|
||||
sock.destroy();
|
||||
}, timeoutMs);
|
||||
const finish = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
fn();
|
||||
};
|
||||
sock.setNoDelay(true);
|
||||
sock.on("connect", () => {
|
||||
sock.write(`GET ${path} HTTP/1.0\r\nHost: ${host}\r\nConnection: close\r\n\r\n`);
|
||||
});
|
||||
sock.on("data", (c: Buffer) => chunks.push(c));
|
||||
sock.on("error", (err) => finish(() => reject(err)));
|
||||
sock.on("close", () => finish(() => resolve(Buffer.concat(chunks).toString("latin1"))));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a raw reply into its HTTP status and body. A reply that starts with a status
|
||||
* line is real HTTP (status + headers, body after the blank line); anything else is
|
||||
* the HTTP/0.9-style reply this board sends — the whole thing IS the body, status 200.
|
||||
*/
|
||||
export function parseRawReply(raw: string): { status: number; body: string } {
|
||||
const m = /^HTTP\/\d\.\d\s+(\d{3})[^\r\n]*\r?\n/.exec(raw);
|
||||
if (!m) return { status: 200, body: raw };
|
||||
const sep = raw.search(/\r?\n\r?\n/);
|
||||
const body = sep === -1 ? "" : raw.slice(sep).replace(/^\r?\n\r?\n/, "");
|
||||
return { status: Number(m[1]), body };
|
||||
}
|
||||
|
||||
/** A cell's visible text: inner tags stripped (the board wraps a "Yes" in markup the
|
||||
* "No" cells don't carry), entities and padding normalised, lowercased. */
|
||||
function cellText(inner: string): string {
|
||||
return inner
|
||||
.replace(/<[^>]*>/g, "")
|
||||
.replace(/ /gi, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the status table into boolean flags. Each fault is a `<TD>label</TD>
|
||||
* <TD>Yes|No</TD>` pair (the board pads the value with spaces, and may wrap a fault's
|
||||
* "Yes" in its own tags — 2026-09-09, seen live as "missing coverOpen, paperEnd,
|
||||
* offline" with the cover open, i.e. exactly the Yes cells). Returns only the
|
||||
* recognised fields; a missing field stays undefined so the caller can detect an
|
||||
* unexpected page (fail safe, not a false "ok").
|
||||
*/
|
||||
export function parseStatusPage(html: string): StatusFlags {
|
||||
const out: StatusFlags = {};
|
||||
const rowRe = /<TR[^>]*>\s*<TD[^>]*>([\s\S]*?)<\/TD>\s*<TD[^>]*>([\s\S]*?)<\/TD>/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = rowRe.exec(html))) {
|
||||
if (m[1] === undefined || m[2] === undefined) continue;
|
||||
const key = STATUS_FIELDS[cellText(m[1])];
|
||||
const value = cellText(m[2]);
|
||||
if (key && (value === "yes" || value === "no")) out[key] = value === "yes";
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
class K200lPrinter implements PrinterDevice, MonitorableDevice {
|
||||
readonly driverId = "k200l";
|
||||
/** The print path — the generic ESC/POS device built from the SAME config. */
|
||||
readonly #print: PrinterDevice;
|
||||
readonly #tcp: boolean;
|
||||
readonly #host: string;
|
||||
readonly #httpPort: number;
|
||||
readonly #timeout: number;
|
||||
|
||||
constructor(config: DeviceConfig) {
|
||||
this.#print = escposDriver.create(config) as PrinterDevice;
|
||||
this.#tcp = transportFromConfig(config).kind === "tcp";
|
||||
this.#host = config.host ? String(config.host) : "";
|
||||
this.#httpPort = config.httpPort ? Number(config.httpPort) : 80;
|
||||
this.#timeout = config.timeoutMs ? Number(config.timeoutMs) : 3000;
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
await this.#print.connect();
|
||||
}
|
||||
|
||||
async disconnect(): Promise<void> {
|
||||
await this.#print.disconnect();
|
||||
stubLog(this.driverId, "disconnect");
|
||||
}
|
||||
|
||||
healthCheck(): Promise<DeviceHealth> {
|
||||
return this.#print.healthCheck();
|
||||
}
|
||||
|
||||
printTicket(data: TicketData): Promise<void> {
|
||||
return this.#print.printTicket(data);
|
||||
}
|
||||
|
||||
printReport(report: PrintReport): Promise<void> {
|
||||
return this.#print.printReport(report);
|
||||
}
|
||||
|
||||
printSubscriptionCard(data: SubscriptionCardData): Promise<void> {
|
||||
return this.#print.printSubscriptionCard(data);
|
||||
}
|
||||
|
||||
printReceipt(data: ReceiptData): Promise<void> {
|
||||
return this.#print.printReceipt(data);
|
||||
}
|
||||
|
||||
printWindowChargeNotice(data: WindowChargeNoticeData): Promise<void> {
|
||||
return this.#print.printWindowChargeNotice(data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Live operator-actionable status from the board's own page.
|
||||
* - USB: no page — reachability floor only (ready/offline, never a guessed state);
|
||||
* - board unreachable / timeout → offline (the same signal as a dead printer);
|
||||
* - page reachable but not the status table (wrong path, index served, non-200) →
|
||||
* degraded ("unexpected status page") — never "ready" off a page we didn't read;
|
||||
* - any fault flag true → degraded, with the faults named; otherwise → ready.
|
||||
*/
|
||||
async readStatus(): Promise<PrinterStatus> {
|
||||
const checkedAt = new Date().toISOString();
|
||||
if (!this.#tcp) {
|
||||
const h = await this.#print.healthCheck();
|
||||
return { status: h.status === "ready" ? "ready" : "offline", detail: h.detail, checkedAt };
|
||||
}
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await fetchRaw(this.#host, this.#httpPort, K200L_STATUS_PATH, this.#timeout);
|
||||
} catch (err) {
|
||||
return { status: "offline", detail: (err as Error).message, checkedAt };
|
||||
}
|
||||
const { status, body } = parseRawReply(raw);
|
||||
if (status !== 200) {
|
||||
return { status: "degraded", detail: `unexpected status page (${K200L_STATUS_PATH}: HTTP ${status})`, checkedAt };
|
||||
}
|
||||
const flags = parseStatusPage(body);
|
||||
const missing = EXPECTED.filter((k) => flags[k] === undefined);
|
||||
if (missing.length > 0) {
|
||||
return {
|
||||
status: "degraded",
|
||||
detail: `unexpected status page (${K200L_STATUS_PATH}: missing ${missing.join(", ")})`,
|
||||
checkedAt,
|
||||
};
|
||||
}
|
||||
const faults = EXPECTED.filter((k) => flags[k] === true);
|
||||
return {
|
||||
status: faults.length > 0 ? "degraded" : "ready",
|
||||
...flags,
|
||||
detail: faults.length > 0 ? faults.map((f) => LABELS[f]).join(", ") : undefined,
|
||||
checkedAt,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const httpPortField: ConfigField = {
|
||||
key: "httpPort",
|
||||
label: "Status web port",
|
||||
type: "port",
|
||||
required: false,
|
||||
default: 80,
|
||||
help: "The board's web configurator port; the status page /prt_status.htm is read from it for live monitoring (default 80). TCP only.",
|
||||
};
|
||||
|
||||
/** The generic driver's fields (transport, device path, host, port, role, rank,
|
||||
* timeout) plus the status-page port, placed right after the print port. */
|
||||
function k200lFields(): ConfigField[] {
|
||||
const out = [...escposDriver.configFields];
|
||||
const i = out.findIndex((f) => f.key === "port");
|
||||
out.splice(i === -1 ? out.length : i + 1, 0, httpPortField);
|
||||
return out;
|
||||
}
|
||||
|
||||
export const k200lDriver: PrinterDriver = {
|
||||
id: "k200l",
|
||||
category: "printer",
|
||||
label: "K200L 80mm thermal printer (Xprinter / ICS, USB+LAN)",
|
||||
description:
|
||||
"Xprinter / ICS K200L (XP-K200L) 80mm ESC/POS printer; its LAN board reports itself as 'POS-80' (web configurator at 192.168.123.100:80 from the factory, DHCP off). Prints over raw TCP (port 9100) OR local USB /dev/usb/lp0 — the same bytes as the generic ESC/POS driver. Over TCP the board's /prt_status.htm page gives live paper / cover / cutter / off-line status; over USB there is no page, so it is monitored by reachability only. No auth on the print socket or the web UI — isolate the VLAN.",
|
||||
transports: ["tcp-ip", "usb"],
|
||||
configFields: k200lFields(),
|
||||
create: (c) => new K200lPrinter(c),
|
||||
};
|
||||
@@ -18,6 +18,7 @@ export {
|
||||
hikvisionDriver,
|
||||
dahuaDriver,
|
||||
rongtaDriver,
|
||||
k200lDriver,
|
||||
escposDriver,
|
||||
} from "./drivers/index.js";
|
||||
export { isPrinter, type PrinterRole } from "./drivers/printer-rongta.js";
|
||||
|
||||
Generated
+2
@@ -117,6 +117,8 @@ importers:
|
||||
specifier: ^4.1.9
|
||||
version: 4.1.9(@types/node@25.9.3)(jsdom@25.0.1)(vite@8.0.16(@types/node@25.9.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4))
|
||||
|
||||
apps/trainer: {}
|
||||
|
||||
apps/vision: {}
|
||||
|
||||
apps/web:
|
||||
|
||||
@@ -70,9 +70,9 @@ RESET_ALLOWED=1 DATABASE_URL=/path node packages/db/scripts/reset-db.mjs --all
|
||||
|
||||
| Flag | Wipes | Keeps |
|
||||
| --- | --- | --- |
|
||||
| `--financial` | `ledger_events` (entry/exit/payment/void/shift/cash/anomaly), `device_events`, `snapshots`, subscription **instances** + credentials/plates, `blocklist` | users, devices, config, tariffs, subscription **plans** |
|
||||
| `--config` | `site_config`, `devices`, `setup_state` (→ re-runs first-run setup), tariffs + versions + **drafts**, subscription plans | everything else |
|
||||
| `--users` | `users`, `roles`, `role_permissions`, auth `sessions` | everything else |
|
||||
| `--financial` | `ledger_events` (entry/exit/payment/void/shift/cash/anomaly), `device_events`, `snapshots`, subscription **instances** + credentials/plates, `blocklist`, `carwash_orders`, `carwash_review_outbox` | users, devices, config, tariffs, subscription **plans**, Car Wash master data |
|
||||
| `--config` | `site_config`, `devices`, `setup_state` (→ re-runs first-run setup), tariffs + versions + **drafts**, subscription plans, validation programs, `carwash_prices/categories/services/config` | everything else |
|
||||
| `--users` | `users`, `roles`, `role_permissions`, `role_jobs`, auth `sessions` | everything else |
|
||||
| `--diagnostics` | `app_logs` (the unsigned [[app-logs]] store behind `/setup/logs`) | everything else |
|
||||
| `--all` | every table (blank slate) | — |
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
type: concept
|
||||
tags: [parking, printer, device, monitoring, reliability]
|
||||
sources: []
|
||||
updated: 2026-06-14
|
||||
updated: 2026-09-09
|
||||
---
|
||||
|
||||
# Printer status monitoring
|
||||
@@ -90,3 +90,17 @@ reads. Full repo typechecks.
|
||||
Yes/No and degrade on anything unexpected, so this is a confidence check, not a blocker.
|
||||
- Tying a `degraded`/`offline` entry-dispenser into [[printer-roles-failover]] failover and the
|
||||
(not-yet-built) entry flow's all-printers-down policy ([[device-input-flow]]).
|
||||
|
||||
## K200L — a second status-page board, and a fetch that node:http can't do (2026-09-09)
|
||||
|
||||
The park-buzi printer turned out to be a **K200L** (Xprinter/ICS XP-K200L; LAN board "J-Speed
|
||||
Ethernet WebConfig 1.02", self-named "POS-80"). Its board serves the **same five decoded Yes/No
|
||||
rows** as the Rongta page — under **`/prt_status.htm`**. Two things kept it invisible until now:
|
||||
the Rongta driver only knew `/prn_stat.htm` (so in July the unit was filed as "no status page →
|
||||
generic driver"), and the board's reply carries **no HTTP status line or headers** (HTTP/0.9
|
||||
style), which `node:http` rejects outright and `curl` shows as an empty `000`. The new **`k200l`**
|
||||
driver (`printer-k200l.ts`) prints through the generic ESC/POS path and reads the page over a raw
|
||||
TCP socket, tolerant of both reply shapes; mapping is the Rongta one (unreachable → offline; page
|
||||
not understood → degraded, never ready; any fault → degraded naming it; else ready; USB →
|
||||
reachability floor). Bench-verified: cover open → amber "cover open, paper out, printer off-line".
|
||||
Tests replay the captured headerless reply. Full device notes: [[k200l-printer]].
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
type: concept
|
||||
tags: [parking, device, printer, transport, usb, escpos, provisioning]
|
||||
sources: []
|
||||
updated: 2026-08-30
|
||||
updated: 2026-09-09
|
||||
status: settled
|
||||
---
|
||||
|
||||
@@ -134,6 +134,11 @@ preselecting the first present device; a saved-but-unplugged path stays selectab
|
||||
"saved — not present now"; zero devices found falls back to the free-text path + a check-the-cable
|
||||
hint. The transport option label no longer hardcodes lp0.
|
||||
|
||||
> **Superseded 2026-09-09:** the XP-K200L DOES serve a status page — the same table as the Rongta,
|
||||
> at **`/prt_status.htm`**, without HTTP headers. It now has its own **`k200l`** driver (raw-socket
|
||||
> fetch; LAN = live cover/paper status, USB = reachability floor). See [[k200l-printer]]. The note
|
||||
> below is kept for the record.
|
||||
>
|
||||
> Driver-choice note for this clone: the ICS XP-K200L does NOT serve the Rongta `/prn_stat.htm`
|
||||
> status page (checked on hardware at 10.0.10.11 — print socket 9100 open, status page absent),
|
||||
> so on NETWORK the honest driver is **cashino** (reachability-only monitoring); under `rongta`
|
||||
@@ -174,7 +179,60 @@ itself is redone.
|
||||
target on a policy-driven restart, the container can come back up still bound to the pre-incident
|
||||
view. This matches the exact reported asymmetry (reboot doesn't fix it; explicit restart does).
|
||||
|
||||
**Not yet confirmed on hardware** — this is the leading theory, not a verified root cause. To
|
||||
> **Bench result 2026-09-09 — the re-enumeration hypothesis is FALSIFIED for this unit.** The
|
||||
> failing printer (`1fc9:2016` "POS-80", now on the dev bench, attached to WSL via usbipd) was
|
||||
> cover-cycled while `dmesg -w` and `lsusb` were watched: **nothing** — no disconnect, no
|
||||
> re-enumeration, same bus/device number (a real drop would have shown as a vhci detach, since
|
||||
> Windows sees the bus first). So the device node does NOT change when the cover opens, and the
|
||||
> container `/dev/usb` bind-mount cannot be going stale for that reason. The failure is in how
|
||||
> `usblp` / the app's open-probe reacts to the printer's **error state** (cover-open status),
|
||||
> not in the device node. Next discriminator is the **exact `detail` text** the monitor logged
|
||||
> on park-buzi at the offline transition (`docker logs <stack>-server-1 | grep
|
||||
> 'device-monitor:.*-> offline'`): `EBUSY` = a handle is held inside the server process (usblp
|
||||
> allows ONE opener — candidate: the `withTimeout` open-leak or a close that never returned;
|
||||
> fits "docker restart fixes"), `usb open timeout` = `open()` itself blocks in the kernel, `EIO`
|
||||
> = `usblp_open`'s bidirectional read submit failed (printer endpoint state). The theory below
|
||||
> is kept for the record.
|
||||
|
||||
> **Lab reproduction FAILED to reproduce (2026-09-09, later the same day).** The same printer
|
||||
> unit on the `park-lab` box (a real Linux host, the booth's exact image `stage-2d9bb15`, the
|
||||
> prod compose with the `/dev/usb` bind-mount, the dev DB snapshot with the USB printer added as
|
||||
> `booth-receipt`, cards printed via the subscription "Reprint card" path): paper out → open
|
||||
> cover → load roll → close cover → reprint — **no error, status never stuck offline.** So the
|
||||
> printer, the app's USB transport and the compose wiring are cleared in isolation. What is left
|
||||
> is park-buzi's own environment (kernel/USB stack, the physical USB port/hub/cable/power at the
|
||||
> booth) and/or that container's *history* (weeks of uptime before the first failure — a leaked
|
||||
> handle needs a prior timeout to exist; a fresh container has none).
|
||||
>
|
||||
> **Status: park-buzi closed (staff shortage), everything shut down — evidence pending.** The
|
||||
> evidence is on the booth's DISK and survives shutdown/reboot: Docker keeps the container log
|
||||
> under `/var/lib/docker/containers/<id>/`, the kernel journal is persistent. **The day the box
|
||||
> powers on again (or lands on the bench), pull these FIRST, before deploying anything:**
|
||||
>
|
||||
> ```bash
|
||||
> # 1. the app's own record: the exact error text at every offline/ready transition
|
||||
> docker logs park-buzi-server-1 2>&1 | grep -E "device-monitor:.*printer.*-> (offline|ready)"
|
||||
> # 2. what the HOST kernel saw around those times (usblp errors, resets, disconnects)
|
||||
> sudo journalctl -k --since "-30 days" | grep -i -E "usblp|usb 1-|usb 2-|disconnect|reset"
|
||||
> # 3. the physical path: hub or direct port? (and note which PSU feeds the printer)
|
||||
> lsusb -t
|
||||
> ```
|
||||
>
|
||||
> Reading (1): `EBUSY` = a handle stuck inside the server process (usblp allows ONE opener;
|
||||
> fits "container restart fixes it") → look at the `withTimeout` leak below; `usb open timeout`
|
||||
> = `open()` blocks in the kernel; `EIO` = `usblp_open`'s bidirectional read submit failed
|
||||
> (printer/link state). Reading (2): any `USB disconnect` / `reset` / `usblp1: removed` at the
|
||||
> transition times means the LINK dropped at the booth (cable/port/hub/power) even though the
|
||||
> unit never dropped on the bench.
|
||||
>
|
||||
> **Follow-ups that need no booth (proposed, not built):** (a) `withTimeout` in
|
||||
> `printer-escpos.ts` abandons the FileHandle when an open/write times out — close it when the
|
||||
> underlying promise eventually settles, so a timeout can never leave the node held; (b) make
|
||||
> the monitor self-document the next occurrence: after N consecutive offline polls on a USB
|
||||
> printer, log the errno, `ls -la /dev/usb`, and who holds the node, so the next failure anywhere
|
||||
> in the fleet carries its own diagnosis without a person at the booth.
|
||||
|
||||
**Not confirmed on hardware — and now contradicted by the bench (above).** The original plan to
|
||||
confirm at the next occurrence, BEFORE restarting anything:
|
||||
```bash
|
||||
# host:
|
||||
@@ -197,8 +255,19 @@ whether the Bus/Device number changes.
|
||||
passthrough + a udev rule pinning a stable symlink name — reintroduces the renumbering fragility
|
||||
the directory bind-mount was chosen to avoid, so only worth doing alongside (1)/(2), not instead.
|
||||
|
||||
**Open sub-question — printer identity.** The park-buzi unit shows as "Generic (unknown)" in the
|
||||
app; not yet identified by vendor/product ID. Lab reproduction uses a **RONGTA** unit instead (not
|
||||
**Printer identity — IDENTIFIED 2026-09-09.** The failing unit is on the dev bench: a **K200L**
|
||||
(Xprinter/ICS XP-K200L family — see [[k200l-printer]] for the LAN setup and its status page): USB
|
||||
`1fc9:2016`, product string **"Printer POS-80"** (0x1fc9 = NXP, the printer's USB controller chip;
|
||||
"POS-80" is the generic 80 mm ESC/POS designation — no brand in the descriptor, which is why the app
|
||||
shows "Generic"). Seen via `usbipd list` on the Windows host (busid 8-1). **Dev-bench caveat:** the
|
||||
stock Microsoft WSL2 kernel (6.6.87.2) has `CONFIG_USB_PRINTER` **not set** — usbip/vhci is there,
|
||||
so the printer can be attached and seen by `lsusb`, but no `usblp` → no `/dev/usb/lpN` → the app's
|
||||
USB transport and the container's `/dev/usb` bind-mount cannot be exercised without a custom WSL
|
||||
kernel (`.wslconfig` `kernel=`) built with `CONFIG_USB_PRINTER=y`. Also, through usbip the
|
||||
cover-open disconnect is seen by *Windows* first (usbipd detaches; `--auto-attach` re-exports), so
|
||||
the bench only shows *whether* the device drops off the bus, not the host-kernel/container
|
||||
staleness itself. Previously: the park-buzi unit showed as "Generic (unknown)" in the app; not
|
||||
identified by vendor/product ID. Lab reproduction uses a **RONGTA** unit instead (not
|
||||
the same hardware), so the lab cannot currently reproduce the park-buzi symptom directly — only
|
||||
validate the general re-enumeration mechanism. Commands to identify the real park-buzi printer next
|
||||
time it's reachable via SSH: `lsusb`, `udevadm info -q property -n /dev/usb/lp1`, `udevadm info -a
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
---
|
||||
type: reference
|
||||
tags: [parking, runbook, installation, devices, network, field]
|
||||
sources: []
|
||||
updated: 2026-09-09
|
||||
---
|
||||
|
||||
# Site device installation — know it BEFORE you are standing in the booth
|
||||
|
||||
A field runbook: for every device we deploy, its **factory address and credentials**, the **tool**
|
||||
you need, **what the app configures by itself** at assign time versus **what must be done by hand on
|
||||
the device**, and the **traps already paid for** on park-buzi and the lab. The [[appliance-provisioning]]
|
||||
runbook covers the booth PC (OS, disk, Docker, Periphery); this page covers everything plugged into
|
||||
it. Written 2026-09-09 after an evening lost to a printer whose factory address nobody had written
|
||||
down ([[k200l-printer]]).
|
||||
|
||||
Rule of thumb that explains most of this page: **every field device ships on its own private
|
||||
subnet with DHCP off, and none of them announce themselves.** You bring a laptop that can take a
|
||||
second static address, you put it on the device's factory subnet, you move the device to the site
|
||||
plan, and only then does anything else see it.
|
||||
|
||||
## Before leaving the office
|
||||
|
||||
**Bring**
|
||||
|
||||
- A laptop with an Ethernet port and the right to add a **second static IPv4 address** to it
|
||||
(Windows: adapter → IPv4 → Advanced → add). Under WSL, remember the source-address bug
|
||||
([[wsl-dev-networking]]): after adding a temporary address, `ping` may work while HTTP times out.
|
||||
- The Dingtian reader tool **`QRCode_v1_6_5.exe`** (Windows) — the only way to set a DT-008's IP,
|
||||
server target, prefixes and symbologies. A browser is enough for everything else.
|
||||
- Patch cables, a USB A–B cable (printers), the site's **address plan** (below) filled in, the
|
||||
app **admin** password, and a fresh Komodo **onboarding key** if the booth PC is new
|
||||
([[appliance-provisioning]] §7a).
|
||||
- The **serials** if already known: DT-008 `cjihao` (on the reader's label / in the tool), camera
|
||||
MAC/serial, printer model (label on the bottom — the K200L's own web UI calls it "POS-80").
|
||||
|
||||
**Address plan** — the device VLAN ([[network-isolation]]) is `10.0.10.0/24` on both sites so far,
|
||||
**every device static, DHCP off everywhere**. The convention from park-buzi / park-lab:
|
||||
|
||||
| Address | Device | Factory address it came from |
|
||||
| --- | --- | --- |
|
||||
| 10.0.10.1 | VLAN gateway (switch/router) | — |
|
||||
| 10.0.10.5 | [[dingtian-relay]] board (barriers + inputs) | `192.168.1.100` |
|
||||
| 10.0.10.7 / .8 | [[dingtian-dt008-reader]] entry / exit (**unique IP each**) | `192.168.1.99` |
|
||||
| 10.0.10.9 | entry-dispenser printer (park-buzi: Cashino KP-300H) | *not recorded — fill in* |
|
||||
| 10.0.10.10 | booth-receipt printer (park-buzi: [[rongta-printer]]) | *not recorded — fill in* |
|
||||
| 10.0.10.11 / .7 (lab) | [[k200l-printer]] | `192.168.123.100` |
|
||||
| 10.0.10.12 / .13 | [[lpr-camera]] entry / exit (Hikvision DS-2CD1047G3H-LIU) | `192.168.1.64` (Hikvision default; needs activation) |
|
||||
| 10.0.10.203 | the booth PC on the device VLAN (**the `backendIp` every device pushes to**) | — |
|
||||
|
||||
Fill the real numbers into the site record before you drive; the wizard asks for the booth's push
|
||||
address once and writes it into the Dingtian and the cameras.
|
||||
|
||||
## 1. Dingtian relay board (DT-R004 family) — barriers, button, radar
|
||||
|
||||
**Factory:** IP `192.168.1.100`, web UI on port 80, login `admin` / `admin`, UDP `60000` (binary) /
|
||||
`60001` (string), multicast discovery `224.0.2.11:60000`. See [[dingtian-relay]].
|
||||
|
||||
**By hand, on the device (browser at its factory address):** set the site IP / mask / gateway in
|
||||
the Network page and reboot. That is the only thing you *must* do by hand. Optional but recommended:
|
||||
in the web UI **disable the UDP2 "string" protocol** (a password-less relay-fire path); the app's
|
||||
harden step tries to disable it and **warns if the device refused** — then do it here.
|
||||
|
||||
**Wiring:** button on **I1** (NO contact to GND, idles HIGH, pulls LOW on press); radar dry contact
|
||||
on **I2**; barrier operator's open input on **relay 1** (entry) and **relay 2** (exit); a spare relay
|
||||
for the entry button lamp ([[button-light-indicator]]). One board can carry both barriers; two
|
||||
distant barriers = two boards ([[entry-exit-points]]).
|
||||
|
||||
**What the wizard does on assign** ([[first-run-setup]], [[device-input-flow]]): finds the board by
|
||||
multicast ("Scan for controllers" — laptop/booth must share the L2 segment), checks and clears
|
||||
`input_link_relay` (factory default auto-fires a relay from its input — the app must decide, not the
|
||||
board), sets a random **relay password** (UDP binary), disables every other control channel, rotates
|
||||
the web login, and writes the **input push** URL + per-device Digest credentials so button/radar
|
||||
edges reach the booth PC. You enter the relay map (which relay is entry/exit/both) and the inputs
|
||||
(button → its relay; radar → `presence`, `activeLow` if it idles opposite the button —
|
||||
[[hikvision-radar]]).
|
||||
|
||||
**Traps**
|
||||
|
||||
- The HTTP CGI API is **unauthenticated** on this firmware; `admin/admin` gates only the web page.
|
||||
Never enable `session_en` — it bricks the config API and only a **factory reset** recovers. The
|
||||
VLAN is the boundary, not the login ([[dingtian-relay]] §Hardening).
|
||||
- A relay password mismatch shows as **"offline despite ping"**: the status query is answered only
|
||||
with the right password. Re-assign / re-enter the relay password in the device form.
|
||||
- "Relay test" in Setup pulses real hardware and signs a ledger event — use it to prove wiring,
|
||||
once per relay.
|
||||
|
||||
## 2. Dingtian DT-008 QR + RFID readers
|
||||
|
||||
**Factory:** IP `192.168.1.99`; no web UI — everything is set with **`QRCode_v1_6_5.exe`** over the
|
||||
network. See [[dingtian-dt008-reader]].
|
||||
|
||||
**By hand, in the tool, per reader:**
|
||||
|
||||
1. **Unique device IP** (`.7` entry, `.8` exit). Two readers on one IP was the 2026-06-18
|
||||
"wrong barrier" incident — scans land on the wrong device row.
|
||||
2. **Server IP** = the booth PC (`10.0.10.203`), **server port** = the booth's HTTP port (80 behind
|
||||
the prod proxy); "server language" can stay whatever it is (php/jsp/asp/aspx/cgi are all
|
||||
served — the reader GETs `/qa/mcardsea.<ext>`).
|
||||
3. **Output prefixes:** `QRCode Output Prefix` = `Q:`, `Card Output Prefix` = `K:` (channel
|
||||
tagging — a printed clone of a card cannot pass as the card).
|
||||
4. **Card Input format = `6H`** (defines the UID shape enrolled; changing it later orphans every
|
||||
card).
|
||||
5. **Symbologies: QR + Code128 only**, minimum decode length ≥ 10, checksums on — otherwise low
|
||||
sun through the striped arm produces phantom 6-digit reads (park-buzi, July).
|
||||
6. Note the **serial (`cjihao`)** — the wizard binds the reader by serial, not by IP.
|
||||
|
||||
**In the wizard:** add the reader with its serial, bind it to the controller relay it sits at
|
||||
(direction is inherited from the relay). **Verify:** scan a card — the server log shows
|
||||
`READ serial=… → device=… verdict=… dir=…`; the reader beeps **twice** on accept, once on refuse,
|
||||
and only after the server's reply (no reply = no beep, the scan still happened).
|
||||
|
||||
**Trap:** a factory reset or a swapped unit silently loses items 3–5. Re-apply all of them.
|
||||
|
||||
## 3. Hikvision camera (DS-2CD1047G3H-LIU, AcuSense) — ANPR + snapshots
|
||||
|
||||
**Factory:** `192.168.1.64`, **inactive** until a password is set on first boot (browser at that
|
||||
address or the SADP tool); after activation the login is `admin` / the password you chose. Site
|
||||
convention so far: `admin` / `admin123` on the first units (change per site and record it). See
|
||||
[[lpr-camera]].
|
||||
|
||||
**By hand, on the camera:**
|
||||
|
||||
1. Activate, set the site IP, disable DHCP. Time: NTP off-site is unavailable — the app re-syncs
|
||||
the camera clock from the booth at every offline→ready edge ([[clock-integrity]]).
|
||||
2. **Streams:** the snapshot the app pulls MUST come from the **sub stream** (`102`) — the main
|
||||
stream's ISAPI snapshot returns **503 instantly, always, on this model**. Set the sub stream to
|
||||
the highest resolution the camera allows.
|
||||
3. **Event push:** Event → Motion Detection with the AcuSense **Detection Target = Vehicle** filter
|
||||
ON, "Notify Surveillance Center" on, then Alarm Settings → **Alarm Server** →
|
||||
`http://10.0.10.203/api/devices/hikvision/<deviceId>/event`. The `deviceId` exists only after
|
||||
the wizard assign, so: **assign first, then come back to the camera**. Digest user/password if
|
||||
the firmware allows it (the wizard shows them).
|
||||
4. "Enable Hikvision-CGI" is a different legacy surface — **not** needed for ISAPI.
|
||||
5. **Close the web UI / live view when done.** The camera has few connection slots; a browser left
|
||||
open makes every snapshot pull 503 "Device Busy" ([[lpr-camera]] §503).
|
||||
|
||||
**In the wizard:** driver `hikvision`, host, `admin` password, channel 1, **stream = Sub**, ANPR on,
|
||||
bind to the relay at that barrier, `alarmPushEnabled` on.
|
||||
|
||||
**Verify, do not assume:** drive a car through and look at `GET /api/events` (or the log) for an
|
||||
alarm with `targetType=vehicle`. A camera configured for push that has sent **zero** alarms is
|
||||
broken on its side: pull its *Diagnose Information*; `Main Db is broken` means a corrupt config
|
||||
database → **factory reset**, then redo 1–3 (the Vehicle target filter defaults OFF after a reset).
|
||||
Point the Alarm Server at a dumb HTTP sink on the laptop if you need to see the verbatim body
|
||||
([[lpr-camera]] §"auto-enter but not auto-exit").
|
||||
|
||||
## 4. Radar (vehicle presence at the entry barrier)
|
||||
|
||||
A dry-contact sensor into a Dingtian input, nothing on the network. Check with the board's input
|
||||
status (`00` query → `relays:inputs`) whether it **idles HIGH or LOW**; if it idles opposite the
|
||||
button, set `activeLow` on that input in the wizard, or the gate inverts (tickets only when the
|
||||
lane is empty). It is advisory: it gates the button, it never opens anything ([[hikvision-radar]],
|
||||
[[entry-double-press]]).
|
||||
|
||||
## 5. Printers — three models, one byte stream, different status
|
||||
|
||||
All print the same ESC/POS bytes over **raw TCP 9100** or **USB (`/dev/usb/lpN`)**; what differs
|
||||
is whether the app can see paper/cover state ([[printer-status-monitoring]],
|
||||
[[printer-usb-transport]]). Roles: **entry-dispenser** outside at the lane, **booth-receipt**
|
||||
inside (receipts, subscription cards, Z-reports, and the backup for entry tickets), **wash-desk**
|
||||
if the site has a Car Wash ([[printer-roles-failover]]). Higher `failoverRank` = tried first.
|
||||
|
||||
| Model | Factory network | Config UI | App driver | Live status |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| [[k200l-printer]] (Xprinter/ICS K200L, "POS-80" board; **the park-buzi unit**) | `192.168.123.100/24`, DHCP off | browser, port 80, no login: Configuration → fixed IP → Save → Restart | **`k200l`** | over LAN: cover / paper / cutter / off-line from `/prt_status.htm`; over USB: reachability only |
|
||||
| [[rongta-printer]] (RP-series) | *not recorded — fill in* | status page `/prn_stat.htm` on port 80 | `rongta` | over LAN: full; USB: reachability |
|
||||
| Cashino KP-300H | *not recorded — fill in* | *not recorded* | `escpos` (generic) | reachability only, by design — no trustworthy status source |
|
||||
|
||||
**Prefer LAN over USB** wherever a cable can reach: the booth sees a real amber "cover open, paper
|
||||
out" while a roll is changed, and the `usblp` path (udev rule, node renumbering, the park-buzi
|
||||
"offline after reload" mystery) drops out of the picture. USB needs the appliance's `usblp` +
|
||||
udev rule ([[printer-usb-transport]] §Provisioning) and the printer shows up as `/dev/usb/lpN`,
|
||||
numbered by plug order.
|
||||
|
||||
**Verify:** the wizard's "Test" only *probes* (opens the port / the device node) — it prints
|
||||
nothing. Print something real: a subscription with a QR credential auto-prints its card and has a
|
||||
**Reprint card** button; a payment prints a receipt; a wash till prints slips. Check the Cashino's
|
||||
barcode with a real ticket (the KP-300H garbled overflowing barcodes until the geometry fix).
|
||||
|
||||
## 6. Order of work on site
|
||||
|
||||
1. Address plan on paper; VLAN ports patched; booth PC up with its device-VLAN address.
|
||||
2. Dingtian: factory address → site address (browser) → wire button, radar, barriers.
|
||||
3. Wizard: **controllers first** (the relay map + inputs); "Relay test" each barrier.
|
||||
4. Readers: tool (IP, server, prefixes, format, symbologies) → wizard (serial, bind) → scan test.
|
||||
5. Cameras: activate → IP → sub stream → wizard assign → Alarm Server + Vehicle target → drive-through
|
||||
test → close the browser.
|
||||
6. Printers: site address → wizard (role, rank) → print a card.
|
||||
7. Walk-through: button + radar → ticket; QR entry then exit; RFID; a subscriber's plate at the
|
||||
camera; pay at the booth → receipt → exit; paper reload on each printer while watching the
|
||||
footer.
|
||||
8. Record in the site record: every IP, serial, camera password, printer model, which relay is
|
||||
which, photos of the labels. Remove the temporary laptop addresses. Log out of every device UI.
|
||||
|
||||
## Gaps to fill next time you hold the hardware
|
||||
|
||||
- Factory address and configuration tool of the **Cashino KP-300H** and the **Rongta RP** units
|
||||
(both still unknown here).
|
||||
- The exact screens in `QRCode_v1_6_5.exe` for the reader's IP and server target (a screenshot).
|
||||
- Whether the camera activation was done with SADP or the browser at park-buzi, and the per-site
|
||||
camera password location.
|
||||
- Where the **site record** lives (a page per site under `wiki/entities/`? — park-buzi and park-2
|
||||
have none yet; the Komodo stack env is the closest thing).
|
||||
|
||||
Related: [[appliance-provisioning]] · [[first-run-setup]] · [[device-registry]] ·
|
||||
[[network-isolation]] · [[entry-exit-points]] · [[dingtian-relay]] · [[dingtian-dt008-reader]] ·
|
||||
[[lpr-camera]] · [[hikvision-radar]] · [[k200l-printer]] · [[rongta-printer]] ·
|
||||
[[printer-usb-transport]] · [[wsl-dev-networking]]
|
||||
@@ -2,12 +2,20 @@
|
||||
type: concept
|
||||
tags: [parking, domain, business, pricing, validation, design]
|
||||
sources: [parksql2017-legacy-schema]
|
||||
updated: 2026-06-17
|
||||
updated: 2026-09-08
|
||||
status: open
|
||||
---
|
||||
|
||||
# Validation & Sponsorship — merchant comps, coupons, postpaid B2B
|
||||
|
||||
> **Sponsor accounts superseded (2026-09-08).** The `sponsors` table sketched below — a
|
||||
> counterparty with a stored `balance_minor` and a billing period — is now a special case of the
|
||||
> **[[party-ledger]]** (design): any party (subscriber, hotel, fleet, supplier) with a balance
|
||||
> *derived* from signed `charge` / settlement / `write_off` events, never a stored column. A
|
||||
> postpaid sponsor = a party; each comped stay = a `charge` against it; the monthly invoice = its
|
||||
> statement. The validation *mechanics* (signed validation events on a session) are unchanged
|
||||
> and built ([[validation-discounts]]).
|
||||
|
||||
Builds on [[validation-discounts]] (the signed-event discount mechanism) to add the layer it leaves
|
||||
open: **a sponsor account and postpaid B2B billing.** The driving case — **a nearby business with a
|
||||
postpaid agreement whose customers enter and exit freely, billed to the business monthly.**
|
||||
|
||||
@@ -45,7 +45,12 @@ The wash stream is small; the **entry camera photographs every car**, in exactly
|
||||
classifier is trained on, with zero domain shift. So the booth can also queue **one in N entry
|
||||
vehicle reads** as pure training material: the crop and the camera's class, *no* order, *no*
|
||||
operator, *no* category — same crop-and-blur pipeline, same one-way path, same privacy
|
||||
properties. `CARWASH_REVIEW_ENTRY_SAMPLE=N` (0/unset = off; needs the three upload settings).
|
||||
properties. `CARWASH_REVIEW_ENTRY_SAMPLE=N` = one in N entries; **`1` = every entry, and that is
|
||||
the setting park-2 runs** (user, 2026-09-07: 4 TB on the collector host, bandwidth not an issue —
|
||||
the only limit was ever the reviewer's time; the reviewer labels what they have time for, the
|
||||
rest waits and stays useful once a first model exists, as the unlabelled pile it is measured on).
|
||||
0/unset = off; needs the three upload settings. A washed car arrives twice, as an entry sample
|
||||
and as the wash decision — intended, the `kind` keeps them apart.
|
||||
Seam: the core announces every vehicle read (`deviceEvents.emitVehicleRead`, snapshot.ts, entry
|
||||
and exit) and the Car Wash module decides — it samples entry reads in-process (`sampleEntry()`,
|
||||
exactly one in N) and calls `enqueueEntry()`; the core never imports the module. Packages carry
|
||||
@@ -78,6 +83,24 @@ Setup → Car wash show queued / delivered / abandoned + the last error.
|
||||
is off and **nothing is queued** (an unbounded queue nobody drains is worse than none). Set per
|
||||
booth in the Komodo stack env; compose forwards them.
|
||||
|
||||
- **URL by Netbird DNS name** (`http://docker-station.nb.infra:8090/ingest`): the server container
|
||||
runs on the host network in prod, so it uses the booth's resolver and Netbird's DNS answers
|
||||
`*.nb.infra`; a collector that moves address costs no booth change. A failed lookup behaves like
|
||||
a collector outage (defer, backoff). The collector's own `COLLECTOR_BIND` must be the raw overlay
|
||||
**IP** — Docker port bindings take no hostname.
|
||||
- **Secrets: one per booth, two consumers.** `wash_review_token_booth_2` is referenced by the
|
||||
booth's stack as its `CARWASH_REVIEW_TOKEN` *and* by the collector's stack inside
|
||||
`COLLECTOR_BOOTH_TOKENS=booth-2:[[wash_review_token_booth_2]],booth-3:[[…]]` — one value, nothing
|
||||
to keep in sync, rotating a booth touches one secret. (A first cut had one combined secret for
|
||||
the whole list; replaced the same day — rotation was all-or-nothing and the value lived twice.)
|
||||
Token format: opaque, `openssl rand -hex 32`; the collector only demands ≥ 16 chars and the list
|
||||
splits on commas/whitespace, which hex never contains. Never share a token between booths — it
|
||||
is what names the booth. Total Komodo secrets for one booth + the collector: two (the booth's
|
||||
token, the reviewer's password).
|
||||
- **The operator hash needs no variable**: `sha256(boothId + ":" + username)[:16]`, computed on
|
||||
the booth from values already set; the owner recomputes it from the booth's usernames to map a
|
||||
hash back, the collector never can.
|
||||
|
||||
## The collector — skeleton built 2026-09-06 (`apps/collector`)
|
||||
|
||||
A deliberately small Fastify + SQLite service **in this monorepo** (so it imports the payload
|
||||
@@ -105,12 +128,52 @@ Three surfaces, nothing else — it must not grow into a fleet console:
|
||||
/ fraud rate.
|
||||
- **`GET /export/labels.csv`** — reviewed, usable rows: item, booth, crop path, the reviewer's
|
||||
label, the operator's category + classes, the camera's class + confidence, downgraded, at.
|
||||
Crops are not packaged: the phase-B trainer runs **on the same host** (its GPU) and reads them
|
||||
off the volume — `docker-compose.collector.yml` carries the `trainer` seam as a commented
|
||||
`profiles: [train]` one-off job (next increment).
|
||||
Crops are not packaged: the phase-B trainer runs **on the same host** and reads the SQLite
|
||||
+ crops straight off the volume, read-only ([[bodytype-classifier-training]]: CPU-only, the
|
||||
Xeon is enough) — the `trainer` service beside the collector in
|
||||
`docker-compose.collector.yml` (the CSV export stays for a human with a spreadsheet).
|
||||
- **Training section on `/review`** (+ `/api/training/status|jobs|jobs/:id|versions/:v/report`)
|
||||
— a thin proxy, behind the same reviewer login, to the trainer's job API on the compose
|
||||
network (`COLLECTOR_TRAINER_URL`, unset = hidden): labels per class vs the minimum, Train
|
||||
(mode / backbone / floor), the running job's log, the versions with Report / Evaluate /
|
||||
Publish. The collector forwards only a fixed set of paths and knobs; the trainer validates
|
||||
values and answers 409 while a job runs.
|
||||
|
||||
**Where the data lives.** The collector writes to `/data` in its container: `collector.sqlite`
|
||||
and one JPEG per item at `crops/<booth-id>/<item-id>.jpg`. `/data` is the named Docker volume
|
||||
`collector-data` (compose), on the host under Docker's volume directory — normally
|
||||
`/var/lib/docker/volumes/wash-collector_collector-data/_data/` (`docker volume inspect
|
||||
wash-collector_collector-data` confirms). The trainer mounts the same volume read-only at its
|
||||
own `/data`; nothing is copied or exported for training.
|
||||
|
||||
**Deploy notes.** Bind the published port to the host's **Netbird address** (`COLLECTOR_BIND`),
|
||||
never `0.0.0.0` on a host with a public interface; Netbird policy: booths → this host:8090 and
|
||||
nothing else. The host must be onboarded as a Komodo server like the booths. `TAG` is pinned
|
||||
and promoted with the booths (one sha for all stacks) — fine while the collector stays small;
|
||||
its own repo the day it needs its own cadence.
|
||||
its own repo the day it needs its own cadence. Deploy the collector BEFORE a booth that sends a
|
||||
package kind it does not know (a 422 is abandoned, not retried). The export neutralises cells
|
||||
that start like a spreadsheet formula (category/service names are booth-supplied text).
|
||||
|
||||
> **Incident 2026-09-16 — the collector's DB predated the `kind` column; nothing worked for 9
|
||||
> days and nothing said so.** The reviewer opened /review: *Training — trainer not reachable:
|
||||
> fetch failed*. On the host: collector `stage-2d9bb15` **unhealthy** (`/health` → 500 *no such
|
||||
> column: kind*), trainer healthy but every `/readiness` a Python traceback; the volume's
|
||||
> `collector.sqlite` (created 2026-09-07 by the previous build, **0 items**) had the original
|
||||
> column set. `CREATE TABLE IF NOT EXISTS` shapes only a NEW database — an existing volume keeps
|
||||
> its old columns, so every query naming `kind` failed: the collector's health, **every ingest**
|
||||
> (booths would have got 500s and kept retrying — the log shows none ever arrived, a separate
|
||||
> question), and the trainer's readiness. The trainer's stdlib server printed the traceback and
|
||||
> dropped the socket, which the collector could only render as "fetch failed".
|
||||
>
|
||||
> Fixes (same day): `CollectorDb` now **migrates on open** — `PRAGMA table_info` vs a list of the
|
||||
> columns added since the first deploy, `ALTER TABLE … ADD COLUMN` for each missing one (all
|
||||
> nullable or defaulted; **append to that list whenever a column joins the CREATE**); the trainer's
|
||||
> handlers are guarded — an unexpected exception is a **500 JSON** naming the error, never a
|
||||
> dropped connection; the collector's status proxy surfaces the trainer's error text. Rule going
|
||||
> forward: the collector owns the schema; the trainer only reads; a deploy that changes the table
|
||||
> must be accompanied by a migration entry, and the Training section is the first place a
|
||||
> schema/DB mismatch shows — read its error text before suspecting the network.
|
||||
|
||||
**Status (2026-09-07).** Live: the collector runs on `art-docker-station` and park-2 is wired to
|
||||
it (`stage-dbbb051` on both stacks, every entry sampled). The review screen at
|
||||
`http://docker-station.nb.infra:8090/review` is filling; no labels reviewed yet.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
type: reference
|
||||
tags: [parking, dev-environment, networking, wsl, troubleshooting]
|
||||
sources: []
|
||||
updated: 2026-06-15
|
||||
updated: 2026-09-09
|
||||
---
|
||||
|
||||
# WSL2 Dev Networking (for device testing)
|
||||
@@ -110,6 +110,15 @@ trap:
|
||||
Verified on hardware (2026-06-15): after the hook, `10.0.10.121` pings and the real [[lpr-camera]]
|
||||
Hikvision driver pulls a snapshot with **no** source-forcing (`localAddress` becomes optional).
|
||||
|
||||
> **It bit again, 2026-09-09 — and the fix was pinned to the wrong NIC.** Configuring the
|
||||
> [[k200l-printer]] meant adding `192.168.123.101` beside `10.0.10.203` on the mirrored LAN NIC;
|
||||
> WSL then sourced 10.0.10.x traffic from the 192.168.123 address: `ping` fine, every HTTP
|
||||
> connect timing out, `ip route get 10.0.10.7` showing `src 192.168.123.101`. `parking-net.service`
|
||||
> was active but pins **`eth1`**, and the mirrored LAN NIC is **`eth0`** on this box now — so the
|
||||
> boot fixer was a no-op. Run `deploy/wsl-fix-route-source.sh eth0` (and fix the unit's argument),
|
||||
> or `curl --interface 10.0.10.203 …` as a one-off. Interface names are not stable across WSL
|
||||
> reboots/NIC changes; the script accepts the NIC as an argument for exactly this reason.
|
||||
|
||||
## On the real appliance: multi-subnet is a deployment config, not a WSL hack
|
||||
|
||||
Production is a **dedicated hardened Linux appliance** ([[disk-os-hardening]]), so the WSL story
|
||||
|
||||
@@ -14,6 +14,8 @@ parking appliance. Written from the **first real provisioning, 2026-06-23** (har
|
||||
actual hardware, including the firmware-specific workaround. Companion to [[disk-os-hardening]] (the
|
||||
*why*), [[tpm]] (TPM analysis), [[container-deployment]] (the images), and
|
||||
[[fleet-deployment-komodo]] (the deploy control plane this runbook's §7 uses).
|
||||
**The devices plugged into the booth** (relay board, readers, cameras, radar, printers — factory
|
||||
addresses, tools, hand steps, traps) have their own field runbook: [[site-device-installation]].
|
||||
|
||||
> ⚠ This box is the [[threat-model|outsider-with-the-box]] defence. The load-bearing anti-fraud
|
||||
> control is still [[reconciliation]] over the [[append-only-event-chain|signed chain]] — disk
|
||||
@@ -312,6 +314,12 @@ sudo loginctl enable-linger admin # so the user service starts at boot witho
|
||||
**Verify:** `systemctl --user status periphery` → active; the server **`park-buzi`** appears and
|
||||
goes **OK/green** in Core → Servers. Then **delete the onboarding key**.
|
||||
|
||||
> **After a reboot: `Unit periphery.service not loaded` (park-lab, 2026-09-09).** The unit
|
||||
> existed but had never been **enabled**, so nothing started it at boot and `reset-failed` /
|
||||
> `restart` had nothing to act on. Fix: `systemctl --user daemon-reload && systemctl --user enable
|
||||
> --now periphery`. Add `enable --now` to the install sequence above whenever the installer's own
|
||||
> enable did not stick (check with `systemctl --user is-enabled periphery` before leaving).
|
||||
|
||||
**➜ Next step is §7b below — the Stack itself is not deployed yet.** A green Server in Core just
|
||||
means the agent connected; it runs nothing until you add the Registry/Git accounts and deploy.
|
||||
|
||||
@@ -350,6 +358,12 @@ docker exec -it -e ADMIN_USER=admin -e ADMIN_PASS='<strong-pw>' \
|
||||
park-buzi-server-1 node scripts/seed-admin.mjs
|
||||
```
|
||||
|
||||
The container is named `<stack>-server-1` (compose project = the Komodo stack name: `park-2-server-1`
|
||||
on park-2; `docker ps` confirms). Leave `ADMIN_USER`/`ADMIN_PASS` off and the script prompts
|
||||
(Enter = `admin`) — preferred on a shared shell, the password never enters history. Idempotent: an
|
||||
existing username is left alone unless `FORCE=1` (§7e). After a `--users`/`--all` reset (§7d) run it
|
||||
again — it recreates the built-in `admin` role row the reset removes.
|
||||
|
||||
> **Secrets-on-disk note.** The generated `.env` lands on the booth with **cleartext** secrets
|
||||
> (compose needs real values). That's why the disk is LUKS-encrypted (§3–4) and keys are per-booth
|
||||
> — the encryption is the control, and a single-booth compromise leaks only that booth's key. See
|
||||
@@ -415,6 +429,14 @@ docker exec -it \
|
||||
> it) **and** you type the DB filename to confirm (`parking.sqlite`; `--yes` skips that for scripted
|
||||
> setup only). It is a **training/demo** tool — never run on a production booth's data.
|
||||
|
||||
> **Drift caught 2026-09-07:** the Car Wash module (six `carwash_*` tables) and `role_jobs` had
|
||||
> landed without a category, so the guard refused every reset on a booth carrying them. Categorised
|
||||
> now — orders + the review outbox under `--financial`, prices/categories/services/config under
|
||||
> `--config`, `role_jobs` under `--users` — and verified `--all` on a freshly migrated DB. The script
|
||||
> ships **inside the server image**, so a booth runs the fixed version only from the next deployed
|
||||
> tag; until then `docker cp` the file from the repo into the container at
|
||||
> `/app/node_modules/@parking/db/scripts/reset-db.mjs` and run the same command.
|
||||
|
||||
After `--users`/`--all` (users cleared), re-seed the first admin exactly as in §7b
|
||||
(`docker exec … node scripts/seed-admin.mjs`) so someone can log back in. Since 2026-07-06 the seed
|
||||
script **self-heals the built-in `admin` role row** that this reset also wipes — before that fix the
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
---
|
||||
title: Body-type classifier (phase B) — training path and hardware
|
||||
type: decision
|
||||
status: decided 2026-09-07; BUILT 2026-09-07 (trainer + vision stage); first real run waits for labels
|
||||
related: [vision-review-outbox, opencv-anpr-service, venue-modules, fleet-deployment-komodo, vision-service-packaging, technology-stack]
|
||||
---
|
||||
|
||||
# Body-type classifier (phase B) — training path and hardware
|
||||
|
||||
The Car Wash category suggestion needs SUV vs sedan, which the phase-A COCO detector cannot give
|
||||
([[opencv-anpr-service]] §Vehicle body type). Phase B is a **classifier over the detector's crop**,
|
||||
trained on the reviewer's labels gathered through the [[vision-review-outbox]]. Decided with the
|
||||
user on 2026-09-07 (morning), **built the same day** once the user said "build the trainer for the
|
||||
Xeon". This page is the loop as built; what is still outstanding is at the end.
|
||||
|
||||
## The loop (as built)
|
||||
|
||||
Each step is a place where a person decides. Nothing here runs on its own.
|
||||
|
||||
1. **Train** — `apps/trainer` (`parking-trainer`, Python/uv like the vision service; its own
|
||||
image `parking-trainer`, the `trainer` service beside the collector on the reviewer's host —
|
||||
never a booth service). **Started from the collector's UI:** the Training section of
|
||||
`/review` (readiness, a Train button with mode / backbone / floor, the live log, the
|
||||
versions with Report / Evaluate / Publish) drives a small job API the trainer serves on the
|
||||
compose network (`serve`; stdlib HTTP, one job at a time, each job the CLI as a subprocess
|
||||
with its log persisted under `/out/jobs/`). The collector proxies it behind the reviewer's
|
||||
login; the trainer is never published. `train` reads the collector's `collector.sqlite` and `crops/` **straight off the volume**
|
||||
(read-only), takes only reviewed, usable rows (the operator's pick and the camera's class are
|
||||
never labels), **splits by TIME** (validation = the newest 20 % by *time seen*, so the number
|
||||
reflects tomorrow's traffic), drops classes with fewer than `--min-per-class` (20) labels from
|
||||
that run and reports them, weighs the loss by damped inverse frequency (√, mean 1 — full
|
||||
inverse over-corrects on small sets), trains, and exports. Two modes:
|
||||
- `--mode features` (default): the ImageNet backbone is **frozen**; every crop's feature
|
||||
vector is cached on disk (`<out>/cache/features-<backbone>-<size>.npz`, keyed by item id),
|
||||
and only a linear head is trained — minutes for thousands of crops, **seconds** to retrain
|
||||
when labels arrive (only new crops go through the backbone).
|
||||
- `--mode finetune`: warm-starts the head the same way, then unfreezes everything with light
|
||||
label-preserving augmentation (flip, mild zoom, brightness/contrast) — the step when the
|
||||
cheap mode plateaus.
|
||||
Backbones: `resnet18` (default), `mobilenet_v3_small`, `efficientnet_b0` — torchvision,
|
||||
BSD-3, and the ImageNet weights ship under the same licence (the licence rule applies to
|
||||
weights as much as code). CPU-only PyTorch from PyTorch's own wheel index (`tool.uv.index`).
|
||||
2. **Evaluate before anything ships** — every run writes `report.md` (accuracy, macro recall,
|
||||
per-class recall/precision, confusion matrix, dropped classes, loss weights, agreement with the
|
||||
detector's coarse class) and `metrics.json`. The job **refuses to write the model** below
|
||||
`--min-accuracy` (default 0.85) — exit 3, report still written — and also withholds it if the
|
||||
exported ONNX disagrees with the torch model on validation (< 99 % argmax agreement). Exit 2 =
|
||||
not enough labels (fewer than two classes clear the minimum). Later, `evaluate --model …`
|
||||
scores a shipped model against labels **reviewed after it was trained** (a clean held-out
|
||||
check) and prints its class histogram + detector agreement over the **unlabelled** pile — the
|
||||
ongoing drift check without labelling everything, which is why every entry is sent
|
||||
([[vision-review-outbox]] §The entry stream). 85–95 % on frontal gate views is the
|
||||
expectation once tuned; enough to *flag*, never to *bill*.
|
||||
3. **Publish** — weights are not code and do not live in git. `publish <version dir> --url
|
||||
https://git.infra.msai.al/api/packages/mca/generic/parking-bodytype` PUTs the four files as a
|
||||
**Gitea generic package** version (token with `package:write`, `TRAINER_PUBLISH_TOKEN`).
|
||||
4. **Bake** — `apps/vision/models/bodytype.version` (tracked in git, empty today) **pins** the
|
||||
version the vision image carries. The Dockerfile fetches `bodytype.onnx` + `bodytype.json`
|
||||
from the package registry at build (auth via a BuildKit secret `bodytype_auth` = the
|
||||
registry user's credentials, never a layer); **a pinned version that cannot be fetched fails
|
||||
the build** — the image must carry what git says it carries; empty pin = no classifier, phase
|
||||
B off, build passes. The pin is a normal commit: reviewable, revertible.
|
||||
5. **Deploy** — a TAG bump on the booth's stack. **A booth gets a model the way it gets code**: a
|
||||
pinned release you can see and roll back. No runtime model fetch (air-gapped appliance,
|
||||
read-only model path — [[vision-service-hardening]]).
|
||||
|
||||
Retrain when the labels have grown meaningfully (every few hundred new verdicts at first). First
|
||||
run needs roughly **200 reviewed crops per class that matters** (Vetura and SUV at least) — until
|
||||
then `inspect` says `ready: false` and `train` exits 2.
|
||||
|
||||
## The contract between trainer and booth
|
||||
|
||||
The trainer and the vision service share **no code** (different packages, different images), so
|
||||
the preprocessing contract is **data**: the `bodytype.json` sidecar (`format:
|
||||
parking-bodytype/1`) carries version, the class list (a subset of the shared vocabulary, in
|
||||
vocabulary order), `input_size` (224), `crop_margin` (0.08 — the same as the outbox's
|
||||
`makeReviewCrop`), colour order, resize method, backbone, mode, label counts and the validation
|
||||
metrics. Both sides cut the detector's box + margin, blur the plate strip, squash-resize with
|
||||
OpenCV `INTER_AREA` (the crop *is* the vehicle; no centre-crop that loses a bumper), and feed raw
|
||||
RGB 0–255 float; **normalisation lives inside the ONNX graph**, so a consumer cannot get the
|
||||
constants wrong. Verified on 2026-09-07: a trainer-produced model loaded by the vision service's
|
||||
`BodyTypeClassifier` gives identical classes and probabilities (< 1e-4) to the trainer's own
|
||||
`OnnxClassifier` on the same crops.
|
||||
|
||||
On the booth ([[opencv-anpr-service]] §Phase B): `RefinedVehicleDetector` runs the classifier
|
||||
only when the detector said `car` **or** a class the classifier trained on; a truck or bus it
|
||||
never saw is left alone (its softmax on an unknown thing means nothing). Below
|
||||
`VISION_VEHICLE_CLASSIFIER_MIN_CONFIDENCE` (0.6) the detector's class stands. `vehicle.
|
||||
detector_class` records the coarse class whenever the stage ran; `model_version` reads
|
||||
`…+yolox:…+bodytype:<version>`. The flag on the desk, the mapping chips, the threshold — nothing
|
||||
downstream changed: the vocabulary already held sedan/hatchback/suv/minivan/pickup.
|
||||
|
||||
## One model for the fleet, not one per site (user asked, 2026-09-07)
|
||||
|
||||
The trainer pools **every booth's** reviewed labels into one training set (no per-booth filter),
|
||||
and one `bodytype.version` pin bakes one model into the one vision image every booth runs. Body
|
||||
type is a property of the car, not the site; pooling is what makes 200 crops per class reachable;
|
||||
and a single pinned version is the whole "a booth gets a model the way it gets code" idea. What
|
||||
*is* per site stays in Setup: the class→category mapping and the flag threshold — the model says
|
||||
"suv", the site decides what an SUV costs and when a downgrade is worth flagging.
|
||||
|
||||
Where a site can still differ is the **camera** (mount height, angle, lens), not the cars. Every
|
||||
crop carries its booth id, so the report can break accuracy down per booth — not in the report
|
||||
today; add it once a second site sends labels. A booth filter in the trainer and a second pin
|
||||
would only be built on evidence that a site's view needs its own model.
|
||||
|
||||
## Secrets and access (2026-09-07)
|
||||
|
||||
- **`TRAINER_PUBLISH_TOKEN`** — a Gitea access token with the `write:package` scope, used by the
|
||||
`publish` command and nothing else, to PUT a passing model's files into the generic package
|
||||
`mca/parking-bodytype`. Training, `inspect` and `evaluate` need no token; leave it unset until
|
||||
the first model passes the floor. Create it under a user who can write packages in the `mca`
|
||||
org, store it as the Komodo secret `gitea_package_write_token`, uncomment the line in the
|
||||
`wash-collector` stack.
|
||||
- **Read side** — the CI build fetches the pinned version with the existing registry
|
||||
credentials (`REGISTRY_USERNAME:PASSWORD` as the BuildKit secret `bodytype_auth`); if the
|
||||
package is org-private that user needs package read, which the Docker-registry user already
|
||||
has in Gitea.
|
||||
|
||||
## Hardware (decided 2026-09-07)
|
||||
|
||||
What the owner has: an **NVIDIA Quadro FX 3800** (in hand, not installed), and in
|
||||
`art-docker-station` an **Intel Xeon E3-1225 v5** (4 Skylake cores, AVX2, no AVX-512) with the
|
||||
**Intel HD P530** iGPU.
|
||||
|
||||
- **Quadro FX 3800 — stays in the drawer.** 2009, GT200, compute capability 1.3, 1 GB. CUDA dropped
|
||||
that generation in 2015; no PyTorch build of the last decade can use it. Installing it buys a
|
||||
heater and a driver problem.
|
||||
- **HD P530 — not for training.** Usable for *inference* via OpenVINO, irrelevant here: inference
|
||||
runs on the booths' CPUs, which already do YOLOX in ~250 ms.
|
||||
- **The Xeon does the job.** The problem is small (a few thousand 224-px crops, ten classes, a
|
||||
small pretrained backbone): features mode in minutes, a full fine-tune in roughly an hour with a
|
||||
mobile-sized backbone. Training is occasional and unattended, and the data is already on that
|
||||
host, so nothing moves.
|
||||
- **Consequences for the build (done):** the trainer image is **CPU-only PyTorch** (torch
|
||||
2.14+cpu, ~200 MB of wheels, not the ~5 GB CUDA build); the `trainer` service in
|
||||
`docker-compose.collector.yml` is real — always on, serving the job API, no device
|
||||
reservation (one block to add if a modern card ever lands; the trainer would pick up CUDA),
|
||||
the collector volume mounted read-only, models/reports/logs in its own `trainer-out` volume.
|
||||
- **If faster is ever wanted:** a used mid-range card of the last few generations (~€200) turns
|
||||
the hour into a minute, given a slot and a PSU. **Renting a cloud GPU is rejected**: the crops
|
||||
would leave the premises, and even scrubbed of plates and site that runs against the whole
|
||||
privacy design of the outbox.
|
||||
|
||||
## Running it
|
||||
|
||||
From the collector's `/review` page, Training section: **Train** (mode, backbone, floor) when
|
||||
readiness says enough labels; watch the log; read the report under Versions; **Evaluate** a
|
||||
written version against labels reviewed since; **Publish** it (needs `TRAINER_PUBLISH_TOKEN`
|
||||
in the `wash-collector` stack — commented until the first publish). Then, in git: write the
|
||||
version into `apps/vision/models/bodytype.version`, commit, let the build produce the image,
|
||||
bump the booth's `TAG`. The pin stays a commit on purpose — it is the deploy control.
|
||||
|
||||
The CLI is still there for debugging, inside the running container:
|
||||
`docker compose -f docker-compose.collector.yml exec trainer parking-trainer inspect`.
|
||||
|
||||
## Operating notes (2026-09-07)
|
||||
|
||||
- **First deploy (`stage-f7a262a`) shipped the trainer as a compose *profile*** — a one-off
|
||||
job the owner had to start by hand with `docker compose … --profile train run …` from
|
||||
wherever Komodo's periphery had cloned the repo (`/etc/komodo/stacks/wash-collector/`).
|
||||
The user rightly called that "not so smart": the host runs a periphery, and the reviewer is
|
||||
already in the collector's UI. **Superseded the same day:** the trainer is now a
|
||||
**service** (`restart: unless-stopped`, the `serve` command) and the collector's
|
||||
`/review` page carries the Training section. A deploy starts both containers; `docker ps`
|
||||
shows two.
|
||||
- **Why not a Docker socket in the collector** (the other way to a button): it would hand
|
||||
root on the host to a service that accepts uploads from booths — the party the
|
||||
[[threat-model]] distrusts. The job API keeps the trainer a normal container with a
|
||||
read-only data mount and its own `trainer-out` volume.
|
||||
- **park-2 does not need a bump** until a model is pinned: the vision image ships with an
|
||||
empty `bodytype.version`, phase B off, nothing for a booth to gain.
|
||||
- **Reviewing is the bottleneck**: the Training section shows labels per class against the
|
||||
minimum and keeps Train disabled until two classes clear it.
|
||||
|
||||
**"Training — trainer not reachable: fetch failed" (2026-09-16).** Not a network problem: the
|
||||
trainer answered `/health` but its `/readiness` crashed on the collector's DB (a volume from before
|
||||
the `kind` column) and the stdlib server dropped the socket without a reply. Since the fix the
|
||||
trainer answers **500 JSON with the error** and the collector shows that text; a genuine network
|
||||
failure still reads "fetch failed" / ECONNREFUSED. See [[vision-review-outbox]] §Incident 2026-09-16.
|
||||
|
||||
|
||||
## Packaging rule (same as the vision service)
|
||||
|
||||
Core deps are light (numpy, opencv-headless, onnxruntime): `inspect`, `evaluate`, the data and
|
||||
report code, and the tests run with `uv sync --frozen` alone — **CI syncs without the `train`
|
||||
extra** ([[vision-service-packaging]]); the torch tests `importorskip`. The image bakes
|
||||
`--extra train` and pre-warms the resnet18 + mobilenet_v3_small ImageNet weights so a run needs
|
||||
no network. `pnpm turbo run lint test` covers `@parking/trainer` through the same package.json
|
||||
shim pattern (workspace count 7→8).
|
||||
|
||||
## Outstanding
|
||||
|
||||
- **The first real run** — waits for ~200 reviewed crops per class on the collector (reviewing
|
||||
is the bottleneck now, not code).
|
||||
- **Secrets on the reviewer's host** — a Gitea token with `package:write`
|
||||
(`gitea_package_write_token`) for `publish`; the CI registry user must be able to *read* the
|
||||
generic package (it passes its credentials as the build secret).
|
||||
- **Tuning knobs after the first report** — the floor, `--min-per-class`, whether finetune beats
|
||||
features on this camera. The report decides, not a guess.
|
||||
@@ -484,3 +484,16 @@ booth, `pkexec dpkg -i`, polkit dialog, relaunch, badge shows 0.1.7). The prompt
|
||||
- **Operator-facing consequence:** the in-app prompt now says the install needs the
|
||||
administrator password (i18n `update.prompt`, en + sq). An operator who accepts and can't
|
||||
authenticate simply stays on the current version; nothing breaks, and the failure is logged.
|
||||
|
||||
### v0.2.0 — the first feature release of the desktop bundle (2026-09-07)
|
||||
|
||||
Every tag from v0.1.0 to v0.1.7 was a desktop-shell fix (origins, cookies, WS tickets, the
|
||||
updater manifest). Since v0.1.7 the SPA the bundle carries (`frontendDist: ../../web/dist`)
|
||||
gained the venue-module registry, the Car Wash module with per-till shifts and the wash-desk
|
||||
printer role, roles that remember their jobs with signed edits, the advisory vehicle category
|
||||
from the entry camera, the review outbox status in Setup, and the two-column Car Wash setup —
|
||||
31 commits, none of them shell fixes. Under 0.x that is a **minor** bump, not a patch: **v0.2.0**.
|
||||
`tauri.conf.json` now says 0.2.0 too (the release workflow still rewrites it from the tag, so
|
||||
the file only matters for local bundles). The README's release gate — run the real bundle, LIVE,
|
||||
one mutation, a frontend log row — is still the step between the tag and the push of the tag.
|
||||
|
||||
|
||||
@@ -158,6 +158,12 @@ collector ([[vision-review-outbox]]) runs on the reviewer's GPU host as its own
|
||||
(`wash-collector`, `server = "art-docker-station"`, `file_paths = ["docker-compose.collector.yml"]`).
|
||||
Same repo, branch and pinned `TAG` promotion, its own secret references, and — because a stack
|
||||
names its compose files — nothing booth-side lands on that host and nothing of it on a booth.
|
||||
The same stack carries the phase-B **trainer** as a second service ([[bodytype-classifier-training]]):
|
||||
a deploy starts both, `docker ps` shows two containers, and the trainer is driven from the
|
||||
collector's UI, never from the host's shell (a first cut as a compose *profile* run by hand was
|
||||
replaced the same day — the host runs a periphery, nobody should be typing compose there). Its two env lines (`TRAINER_OUT`, the
|
||||
`TRAINER_PUBLISH_TOKEN` secret reference) stay commented in `resources.toml` until the first
|
||||
publish.
|
||||
|
||||
## Open / not yet done
|
||||
|
||||
@@ -192,7 +198,7 @@ Second `[[stack]]` in `komodo/resources.toml`: **`park-lab`** (server = the lab
|
||||
|
||||
| Stack | compose branch | image tag | secrets |
|
||||
| --- | --- | --- | --- |
|
||||
| park-lab | `dev` | **moving `dev`** (a lab may float) | `park_lab_*` |
|
||||
| park-lab | `dev` → **`stage` (2026-09-09)** | ~~moving `dev`~~ → **pinned to the booth's `stage-<sha>` under reproduction** (2026-09-09: the lab re-joined the fleet to reproduce the park-buzi printer cover-open bug, so it must run the booth's exact image; review outbox under its **own** id `booth-lab` since 2026-09-16 — never a booth's, so bench crops stay separable per booth on the collector) | `park_lab_*` |
|
||||
| park-buzi | `stage` | pinned `stage-<sha>` | `park_buzi_*` |
|
||||
|
||||
The three knobs are independent per stack — the ResourceSync's own branch only governs where the
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
type: decision
|
||||
tags: [parking, decisions, open]
|
||||
sources: [parking-system-architecture]
|
||||
updated: 2026-09-04
|
||||
updated: 2026-09-08
|
||||
status: open
|
||||
---
|
||||
|
||||
@@ -148,3 +148,11 @@ procurement. (See [[parking-system-architecture]] §10.)
|
||||
already carried `subscription:*` (stale note); roles now remember the jobs they follow and
|
||||
a grown job is re-applied with one click, never silently; every role edit is signed as a
|
||||
`config_change`. **Settled** — details on [[venue-modules]] §"Permissions matrix" Status.
|
||||
17. **Party ledger — receivables & payables across modules.** _(Raised by the user, 2026-09-08.)_
|
||||
Postpaid [[subscription]]s, hotel guest-nights billed to the hotel, Car Wash fleet deals on
|
||||
account, and supplier/utility bills all need "who owes whom". Designed as a **counterparty
|
||||
sub-ledger** — parties + signed `charge` / settlement / `write_off` events, balance derived,
|
||||
aging + statements, CSV for the accountant — see [[party-ledger]] (design only, not built).
|
||||
Interacts with #9 (a statement is **not** a fiscal invoice; fiscalisation is off-appliance)
|
||||
and #8 (one currency per party until FX exists). Also reopened on the subscription page: a
|
||||
**renewal is currently off-book** (an edit, no `payment`).
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
---
|
||||
type: decision
|
||||
tags: [parking, decisions, open, finance, ledger, modules, subscriptions, carwash]
|
||||
sources: []
|
||||
updated: 2026-09-08
|
||||
status: open
|
||||
---
|
||||
|
||||
# Party ledger — who owes the site, and whom the site owes
|
||||
|
||||
**Design only (2026-09-08). Nothing built.** Captured from a design conversation with the user; to be
|
||||
refined before any code. The trigger was the [[subscription]] billing redesign: as soon as a
|
||||
subscription can be **postpaid**, the site is *collecting a debt*, and the user immediately listed
|
||||
three more debtors/creditors that need the same treatment. So this is not a subscription feature
|
||||
— it is a **counterparty sub-ledger** that subscriptions, hotels, fleets and suppliers all sit on.
|
||||
|
||||
## The problem stated
|
||||
|
||||
The user's constraints, verbatim in spirit:
|
||||
|
||||
1. **Postpaid subscriptions** — the subscriber pays at the start or end of a month; the site must
|
||||
see what is unpaid.
|
||||
2. **Hotels** — occasional daily access for a hotel's guests, billed to the hotel, not the guest.
|
||||
3. **Car Wash fleet deals** — the wash cleans a company's cars; payment is due per period; the
|
||||
*site* collects the debt.
|
||||
4. **Car Wash suppliers and utility bills** — the wash needs to see what it has paid and still owes
|
||||
its suppliers (detergent, water, electricity).
|
||||
5. **The admin needs one view of uncollected dues: who owes what to the park.**
|
||||
|
||||
Today none of this is modelled. Money exists in exactly two shapes: a signed `payment` at a till
|
||||
([[shift]], [[append-only-event-chain]]) and a drawer voucher (`cash_in` / `cash_out`). Neither
|
||||
names a *counterparty*, so "who owes whom" cannot be asked. The earlier
|
||||
[[validation-sponsorship]] page sketched a `sponsors` table with a stored `balance_minor` for the
|
||||
postpaid-merchant case; this page **supersedes that sketch** with something general.
|
||||
|
||||
## The decision (proposed)
|
||||
|
||||
Add **one core concept, once**: a **party** with an **account**, and three signed ledger event
|
||||
types that move that account. Modules (Parking, Car Wash, later Bar — [[venue-modules]]) append
|
||||
charges against parties; the core owns the party master data, the balance derivation, the
|
||||
statement and the aging report. No module keeps its own receivable.
|
||||
|
||||
### Party (core master data, module-agnostic)
|
||||
|
||||
A party is any legal or natural person the site has money dealings with — a subscriber, a hotel, a
|
||||
fleet company, a utility, a supplier. Mutable master data (like `subscriptions`), soft-deletable
|
||||
([[soft-delete]]):
|
||||
|
||||
```
|
||||
parties id, name, contact, taxId?, currency, kind {customer|supplier|both},
|
||||
creditLimitMinor?, terms {dueDays | calendarDay}, active, deletedAt…
|
||||
```
|
||||
|
||||
A subscriber gets a party row (created with the subscription, or linked to an existing one — a
|
||||
company with five subscriptions is one party). `creditLimitMinor` lets a desk **refuse on-account
|
||||
sales** when the party is over its limit; `terms` gives the default due date of a charge.
|
||||
|
||||
### Three signed event types (the account never stores a balance)
|
||||
|
||||
| Event | Meaning | Payload (signed) | Who appends |
|
||||
| --- | --- | --- | --- |
|
||||
| `charge` | an **accrual** — the party now owes (or is owed) | `partyId, direction {receivable\|payable}, amountMinor, currency, source {module, ref}, periodFrom?, periodTo?, dueAt, operator` | a module (subscription period, guest-night, on-account wash, supplier bill) |
|
||||
| `settlement` | **money moved** against the account | as a **`payment`** at a till (`partyId` + `chargeIds[]` added) for cash/card received; a **`cash_out`** voucher with `partyId` for cash paid out; a `settlement` with `tender: "bank"` and no till for transfers either way | operator at a till / admin for bank |
|
||||
| `write_off` | admin-signed **reduction with a reason** (waived period, disputed night, goodwill) | `partyId, chargeId, amountMinor, reason, operator` | admin only |
|
||||
|
||||
**Balance** per party and currency = Σ charges − Σ settlements − Σ write-offs, derived on read
|
||||
(cached at most), never stored. **Why signed events and not a mutable `balance` column:** the
|
||||
[[threat-model]] adversary is the booth/wash operator. A receivable that lives in a mutable row can
|
||||
be quietly shrunk; a receivable that is a chain of signed events cannot — a statement is
|
||||
re-derivable and **disputable against the chain**, the same guarantee the shift Z-report gives.
|
||||
The one fraud-relevant path is the write-off, which is why it is admin-gated and permanent.
|
||||
|
||||
Reusing `payment` for money received (rather than inventing a parallel type) keeps the drawer,
|
||||
the Z-report and the per-till folds ([[shift]] §Tills) working with **zero new summing surface** —
|
||||
the same reasoning that made a subscription sale a `payment` with a `subscriptionSale` flag
|
||||
([[subscription]] §Collecting the fee).
|
||||
|
||||
### How the four cases land on it
|
||||
|
||||
- **Subscriptions** — the billing-period design ([[subscription]] §Recurring billing) stays exactly
|
||||
as drawn, except a billing period *is* a `charge` against the subscriber's party. Prepaid vs
|
||||
postpaid is only the due-date rule. Paying a period = a till `payment` referencing the charge.
|
||||
- **Hotels** — a subscription-like agreement whose **payer is the hotel party**, postpaid, whose
|
||||
credential is issued per guest for N nights (the existing `"day"` plan). Each guest-night is a
|
||||
charge line; the hotel receives a monthly **statement of nights**. The guest never pays.
|
||||
- **Fleet washes** — the wash order gains a **third `payAt` beside `booth` and `bay`: `account`**.
|
||||
The order is a charge against the fleet party; the wash till's Z-report shows on-account sales
|
||||
as a separate line, *not* cash. Over the credit limit → the wash desk cannot pick `account`.
|
||||
- **Suppliers and utilities** — a bill is a **payable** charge against that party (the wash's
|
||||
detergent supplier, the electricity company). Paying it from the wash till is a `cash_out`
|
||||
voucher that references the bill (the drawer already folds it); paying by bank is a bank
|
||||
settlement. The owner sees what is owed, what was paid, and **from which till**.
|
||||
|
||||
### The admin view
|
||||
|
||||
One report over all parties: name, balance, oldest unpaid charge, **aging buckets** (current,
|
||||
30, 60, 90+ days), drill-down to a **statement** for a period (every charge, settlement and
|
||||
write-off, each linked to its signed event). "Uncollected dues" is a filter on it: receivables
|
||||
with a balance. Payables are the same report with the direction flipped. Everything is a
|
||||
projection over the ledger, like [[reporting-analytics]].
|
||||
|
||||
### Permissions
|
||||
|
||||
New core permissions, in the [[venue-modules]] matrix: `finance:read` (statements, aging),
|
||||
`finance:settle` (record a bank settlement; till settlements ride the existing pay permissions),
|
||||
`finance:writeoff` (admin), `party:manage` (master data). The wash desk sees only *whether* a
|
||||
party is on-account-eligible, never the balance.
|
||||
|
||||
## Where the line is drawn
|
||||
|
||||
This is a **sub-ledger of receivables and payables, not bookkeeping.** No chart of accounts, no
|
||||
profit-and-loss, no VAT computation, no double-entry general ledger. The accountant gets a **CSV
|
||||
export** of charges and settlements per party and period. Two flags before anything is built:
|
||||
|
||||
- **A statement is not a fiscal invoice.** Fiscal receipts/invoices are already
|
||||
[[open-questions]] #9 (tax number, sequential numbering, and — in Albania — fiscalisation).
|
||||
The appliance is [[offline-first]]; fiscal invoicing needs the cloud side
|
||||
([[cloud-service-saas]]) or an external fiscal device. Statements must be **labelled as
|
||||
statements** so nobody mistakes them for invoices.
|
||||
- **Parties are per appliance.** A fleet washing at two sites has two accounts until the
|
||||
PostgreSQL sync target exists. Consolidation is a cloud-side concern.
|
||||
|
||||
Also deliberately **not** built: automatic card charging, dunning sequences, automatic
|
||||
suspension without a grace period. The operator never types a price ([[subscription]] rule).
|
||||
|
||||
## Build order (each step usable on its own)
|
||||
|
||||
1. `parties` + the three event types + the balance/aging/statement report and CSV export.
|
||||
2. Subscription billing periods on top ([[subscription]] §Recurring billing) — the renewal
|
||||
off-book hole closes here.
|
||||
3. `payAt: "account"` on Car Wash orders, with the credit-limit gate and the Z-report line.
|
||||
4. Bills and payables (supplier / utility register; `cash_out` with a bill reference).
|
||||
|
||||
## Open
|
||||
|
||||
- Does a **guest-night** charge get appended at credential issue (N nights known up front) or per
|
||||
actual entry? Issue-time matches the hotel's booking; per-entry matches reality. Lean issue-time,
|
||||
with a void path if the guest never came.
|
||||
- **Currency**: parties carry one currency; a charge in another is refused until the FX question
|
||||
([[open-questions]] #8) is settled.
|
||||
- **Who may create a party** at the wash desk vs. admin only (a fleet deal is a contract, not a
|
||||
walk-in).
|
||||
- Should utility bills live in this app at all, or only supplier bills paid from a till? The user
|
||||
asked for both; the register is cheap, the temptation to grow it into bookkeeping is the risk.
|
||||
- **Reminders to the party** (statement by email/SMS) are off-appliance — same answer as the
|
||||
subscription expiry notice: the operator/owner is notified, the contact is theirs to make.
|
||||
|
||||
Related: [[subscription]] · [[validation-sponsorship]] (superseded sketch) · [[venue-modules]] ·
|
||||
[[shift]] · [[append-only-event-chain]] · [[threat-model]] · [[reporting-analytics]]
|
||||
@@ -87,6 +87,19 @@ The skeleton is **built and wired** (no recognizer models yet):
|
||||
(env `VISION_*`), `schemas.py` (the `/analyze` contract incl. a not-yet-populated `vehicle` field
|
||||
for Job 2), `recognizer.py` (a `Recognizer` **Protocol** + `StubRecognizer` and `FastAlprRecognizer`
|
||||
— the [[device-adapter-pattern]] applied to the model).
|
||||
- **CI runs WITHOUT the extra** (`uv sync --frozen` in ci.yml and build-images.yml): a test that
|
||||
imports numpy/cv2 at module level breaks collection there even though it passes in a local venv
|
||||
that has `alpr`. Rule (2026-09-07, after three red runs): pure post-processing tests get numpy
|
||||
from the **dev group**; anything needing OpenCV uses `pytest.importorskip("cv2")`; the service
|
||||
itself imports both lazily inside functions.
|
||||
- **The same pattern, second package (2026-09-07):** `apps/trainer` (`@parking/trainer`,
|
||||
[[bodytype-classifier-training]]) — light core (numpy, opencv-headless, onnxruntime) + a
|
||||
`train` extra (CPU-only torch/torchvision/onnx/onnxscript from PyTorch's wheel index via
|
||||
`tool.uv.index`); CI syncs without it, torch tests `importorskip("torch")`, the module that
|
||||
imports torch is imported lazily by the `train` command only. Its own image
|
||||
(`parking-trainer`, context `apps/trainer`, uv base image, bakes `--extra train` + the
|
||||
ImageNet backbone weights) is built by build-images.yml beside the other three; both Python
|
||||
contexts now carry a `.dockerignore` (venv/caches/weights out). Workspace count 7→8.
|
||||
- **Light-core, heavy-optional:** core deps boot in **stub mode** (no model download) so `uv sync` +
|
||||
tests work offline; the real stack is the `alpr` extra (`uv sync --extra alpr` →
|
||||
fast-alpr + onnxruntime). `VISION_RECOGNIZER=fast_alpr` switches it on.
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
---
|
||||
type: entity
|
||||
tags: [parking, hardware, printer, escpos, network, usb, status]
|
||||
sources: []
|
||||
updated: 2026-09-09
|
||||
---
|
||||
|
||||
# K200L thermal printer (Xprinter / ICS "XP-K200L") — the park-buzi unit
|
||||
|
||||
An 80 mm ESC/POS receipt printer, **USB + LAN**, sold under several names. Bottom label:
|
||||
*"THERMAL RECEIPT PRINTER — Model: K200L — Paper Width: 80mm — Print Speed: 200mm/s — Power
|
||||
Input: 24V 2.5A — Cash Drawer: 24V 1A — Interface: USB+LAN — Command Support: ESC/POS"*, serial
|
||||
`BLU2107080238`. Identified on the dev bench 2026-09-09; **it is the printer that "goes offline
|
||||
after a paper reload" at park-buzi** ([[printer-usb-transport]] §Field bug). The lab's older
|
||||
"ICS XP-K200L" (10.0.10.11, the 2026-07 USB truncation work) is the same family.
|
||||
|
||||
Three names for one device, all seen on the bench:
|
||||
|
||||
| Where | What it calls itself |
|
||||
| --- | --- |
|
||||
| bottom label | K200L |
|
||||
| USB descriptor (`lsusb`) | `1fc9:2016 NXP Semiconductors Printer-80` / "Printer POS-80" (0x1fc9 = the NXP controller chip; no brand) |
|
||||
| LAN board web UI | "J-Speed Ethernet Interface Module", "Ethernet WebConfig Version 1.02", copyright "POS" |
|
||||
|
||||
The app driver is **`k200l`** ("K200L 80mm thermal printer (Xprinter / ICS, USB+LAN)",
|
||||
`packages/devices/src/drivers/printer-k200l.ts`). It prints through the shared generic ESC/POS
|
||||
path (identical bytes, TCP 9100 or `usblp`) and **adds live status from the LAN board** — see
|
||||
below. Before 2026-09-09 this unit ran on the generic `escpos` driver (reachability only), which is
|
||||
why the app could never show its cover/paper state.
|
||||
|
||||
## Network setup (the evening that was never written down)
|
||||
|
||||
- **Factory address `192.168.123.100/24`, DHCP OFF.** Nothing announces it; the printer just sits
|
||||
there on a subnet nobody uses. To reach it, give the workstation a second address in
|
||||
`192.168.123.0/24` (Windows: adapter → IPv4 → Advanced → add `192.168.123.101`), then open
|
||||
`http://192.168.123.100/`.
|
||||
- The web configurator (port 80, **no authentication**) is a three-frame page: *Information*
|
||||
(`ip_info.htm`: MAC, IP, mask, gateway, DHCP on/off, DHCP timeout), *Configuration*
|
||||
(`ip_config.htm`: DHCP client on/off + timeout, fixed IP / mask / gateway as four octet fields,
|
||||
**Save**, Restore Default, cancel), *Printer Status* (`prt_status.htm`), *Printer Test*
|
||||
(`prt_test.htm`), and a **Restart** button in the menu.
|
||||
- **Set a fixed address on the device VLAN** (park-lab: `10.0.10.7/24`, gateway `10.0.10.1`) →
|
||||
Save → Restart; then remove the temporary `192.168.123.x` address from the workstation. Keep
|
||||
DHCP off — the app addresses printers by IP ([[rongta-printer]] §Deployment).
|
||||
- The self-test page (`prt_test.htm` / the "Print Test Page" button on the status page) prints
|
||||
the current network settings, so a unit with a forgotten address can be read back that way.
|
||||
- The board's HTTP server is **tiny**: the frameset reloads its frames every 1–3 s and the status
|
||||
page every 5 s, and it holds very few connections. **Close the browser tab while the app is
|
||||
polling**, or connections intermittently time out (seen on the bench: `ping` fine, port open,
|
||||
every second HTTP connect hanging while the page was open in a browser).
|
||||
|
||||
> **WSL gotcha while doing this (2026-09-09):** the dev box then carried BOTH `192.168.123.101`
|
||||
> and `10.0.10.203` on `eth0`, and WSL sourced 10.0.10.x traffic from the 192.168.123 address —
|
||||
> the exact [[wsl-dev-networking]] source-address bug, except `parking-net.service` pins `eth1`
|
||||
> and the mirrored LAN NIC is `eth0` now. Symptom: `ping` works, `curl` times out. Run the fix for
|
||||
> `eth0`, or drop the temporary address once the printer is moved.
|
||||
|
||||
## Live status — the `/prt_status.htm` page
|
||||
|
||||
The LAN board serves a five-row table the printer has already decoded from its own sensors:
|
||||
|
||||
```
|
||||
Cover Is Open Yes/No
|
||||
Cutter Error Yes/No
|
||||
Paper End Yes/No
|
||||
Paper Near End Yes/No
|
||||
Printer Off-Line Yes/No
|
||||
```
|
||||
|
||||
A fault is written as **`<FONT color=#ff0000>Yes</FONT>`** while a clear row is a bare, space-padded
|
||||
`No` — the first parser only accepted tag-free cells, so with the cover open the booth showed
|
||||
*"unexpected status page (missing coverOpen, paperEnd, offline)"*: exactly the Yes cells. Fixed
|
||||
the same day (cell text is read with inner tags stripped); the verbatim markup is pinned in the
|
||||
driver's tests.
|
||||
|
||||
**Same rows, same `<TD>label</TD><TD>Yes|No</TD>` shape as the Rongta board's `/prn_stat.htm`**
|
||||
([[printer-status-monitoring]]) — only the path differs, which is why nobody found it in July
|
||||
(the Rongta driver looked for `/prn_stat.htm`, got nothing, and the unit was filed as "serves no
|
||||
status page → generic driver"). Verified on the bench: cover open → `Cover Is Open Yes`, `Paper End
|
||||
Yes`, `Printer Off-Line Yes` within a refresh; cover closed → all `No`.
|
||||
|
||||
**Quirk that needs its own fetch code:** the board's HTTP reply has **no status line and no
|
||||
headers** — the body starts at byte 0 (HTTP/0.9 style). Browsers render it; `curl` reports
|
||||
`000` with an empty body; Node's `http` client rejects it with *"Parse Error: Expected HTTP/, RTSP/
|
||||
or ICE/"*. So the `k200l` driver reads the page over a **raw TCP socket** (`GET … HTTP/1.0`, read
|
||||
until the board closes) and accepts both the headerless reply and a proper one. This is the second
|
||||
reason the K200L has its own driver rather than a path option on the Rongta one.
|
||||
|
||||
Status mapping (mirrors the Rongta driver, [[printer-status-monitoring]]): board unreachable /
|
||||
timeout → **offline**; page reachable but not the table (non-200, the index page) → **degraded
|
||||
"unexpected status page"**, never ready off a page we didn't read; any Yes → **degraded** naming
|
||||
the faults ("cover open, paper out, printer off-line"); all No → **ready**. Over **USB** there is
|
||||
no page: reachability floor only (ready/offline), same as a USB Rongta.
|
||||
|
||||
## What this means for the park-buzi bug
|
||||
|
||||
At park-buzi this unit ran **over USB** on the generic driver, i.e. monitored by "does
|
||||
`/dev/usb/lpN` open". A cover-open / paper-out condition **never showed in the app at all** — the
|
||||
badge stayed green. So the operators' "printer goes offline after reloading paper" was not the
|
||||
cover state being reported; it was a genuine probe failure whose errno is still unread (site shut
|
||||
down — [[printer-usb-transport]] §Field bug has the commands to pull first). Running the unit on
|
||||
**LAN with the `k200l` driver** would give the booth a real amber "cover open / paper out" while
|
||||
the roll is changed, and removes the `usblp` path from the equation altogether — a strong reason to
|
||||
cable it to the device VLAN when the site reopens.
|
||||
|
||||
Related: [[rongta-printer]] · [[printer-status-monitoring]] · [[printer-usb-transport]] ·
|
||||
[[printer-roles-failover]] · [[network-isolation]] · [[wsl-dev-networking]] · [[site-device-installation]]
|
||||
@@ -251,6 +251,10 @@ service's `/health` each tick and shows a **"Vision" chip** in the booth footer
|
||||
|
||||
## Vehicle body type (advisory) — the vehicle stage, phase A (2026-09-06)
|
||||
|
||||
> Phase B (the classifier that knows SUV from sedan), its training loop and the hardware it runs on
|
||||
> are on [[bodytype-classifier-training]] — built 2026-09-07, see §Phase B below; no model is
|
||||
> pinned yet (the stage is off until the first published version).
|
||||
|
||||
`/analyze` populates `vehicle.body_type` + `vehicle.confidence` from the shared vocabulary
|
||||
(car, sedan, hatchback, suv, minivan, pickup, van, truck, bus, motorcycle). Node records it beside
|
||||
the plate and the Car Wash desk pre-selects the category the site maps it to; the operator
|
||||
@@ -290,3 +294,31 @@ read. Composed `model_version` reads `<plate>+yolox:yolox_s.onnx@640`.
|
||||
operator's picks (untrusted — [[threat-model]]) but a trusted reviewer's, gathered through the
|
||||
[[vision-review-outbox]]. Expect 85–95 % on frontal gate views once tuned — enough to flag,
|
||||
never to bill, which is why the flag records and the site threshold exists.
|
||||
|
||||
### Phase B — the body-type classifier stage (built 2026-09-07)
|
||||
|
||||
`vehicle.py` gained a second stage: `BodyTypeClassifier` loads `bodytype.onnx` + its
|
||||
`bodytype.json` sidecar (produced by `apps/trainer`, [[bodytype-classifier-training]] §The
|
||||
contract) and `RefinedVehicleDetector` composes it over the YOLOX detector — the detector still
|
||||
finds and picks the vehicle, the classifier answers on its crop. `crop_vehicle` mirrors the
|
||||
outbox's `makeReviewCrop` (box + the sidecar's margin, plate strip Gaussian-blurred) so the
|
||||
booth sees what the model was trained on; resize is OpenCV `INTER_AREA` at the sidecar's
|
||||
`input_size`, raw RGB 0–255 in, normalisation inside the graph.
|
||||
|
||||
- **Rule:** the classifier runs only when the detector said `car` **or** a class the classifier
|
||||
trained on; a truck/bus/motorcycle it never saw is left alone. Below
|
||||
`VISION_VEHICLE_CLASSIFIER_MIN_CONFIDENCE` (0.6) the detector's class stands. When the stage
|
||||
ran, `vehicle.detector_class` carries the coarse class (Node ignores it today; the collector
|
||||
could show it). `model_version` reads `<plate>+yolox:…+bodytype:<version>`.
|
||||
- **Config:** `VISION_VEHICLE_CLASSIFIER_PATH` (the image sets `/app/models/bodytype.onnx`) and
|
||||
the min-confidence. **Path set but no file = the normal state before the first model** —
|
||||
phase A only, one log line, *no* `/health.detail` error. A file that fails to load IS an error
|
||||
in `detail` (`classifier: …`), and a classifier that throws per frame is caught, noted, and the
|
||||
detector's answer returned — the plate read is never at risk.
|
||||
- **Bake:** `apps/vision/models/bodytype.version` (tracked; empty) pins the published version the
|
||||
Dockerfile fetches from the Gitea generic package registry (BuildKit secret `bodytype_auth`);
|
||||
a pin that cannot be fetched fails the build, an empty pin passes with phase B off.
|
||||
- **Tests** (`tests/test_vehicle.py`): crop margin/clamp/blur, the refine rule (car → suv when
|
||||
confident; unsure → detector's class; unknown bus untouched; a classifier that knows trucks may
|
||||
override a truck), a throwing classifier survives and is reported, missing files → not ready,
|
||||
and the factory skips a missing model without an error.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
type: entity
|
||||
tags: [parking, hardware, printer, device]
|
||||
sources: []
|
||||
updated: 2026-06-19
|
||||
updated: 2026-09-09
|
||||
---
|
||||
|
||||
# Rongta 80mm thermal printer
|
||||
@@ -55,6 +55,10 @@ many ESC/POS-compatible OEM clones that share its firmware). Driver `rongta` in
|
||||
- **booth-receipt** — `10.0.10.10`, inside the booth; receipts, AND the backup that prints
|
||||
the entry ticket if the outside dispenser is offline. This unit is a **Rongta** (`rongta`
|
||||
driver, full status-page monitoring).
|
||||
- **Not a Rongta, its own driver since 2026-09-09:** the **K200L** (Xprinter/ICS XP-K200L family;
|
||||
LAN board calls itself "POS-80") — the park-buzi unit and the lab's 10.0.10.11 unit. Same
|
||||
five-row status table under **`/prt_status.htm`** (not `/prn_stat.htm`), served without HTTP
|
||||
headers, so it has the `k200l` driver with a raw-socket fetch. See [[k200l-printer]].
|
||||
|
||||
## Ticket rendering
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
type: entity
|
||||
tags: [parking, domain, business, subscriptions, identity, pricing]
|
||||
sources: []
|
||||
updated: 2026-06-20
|
||||
updated: 2026-09-08
|
||||
aliases: [subscription-plan]
|
||||
status: open
|
||||
---
|
||||
@@ -90,6 +90,15 @@ subscription row, one window. The amount the operator should collect is **N × t
|
||||
`now` ∈ [validFrom, validTo]** — so a 3-month window simply stays valid for three months.
|
||||
- An explicit **`validTo` override** is still accepted (manual end date) when `months` isn't used.
|
||||
|
||||
> ⚠ **Renewal is OFF-BOOK (found 2026-09-08).** "Renewing is just editing the window" means a
|
||||
> renewal goes through `PUT /api/subscriptions/:id`, which by design **never re-sells and appends
|
||||
> nothing to the ledger**. The first sale was put on the chain on 2026-06-20 precisely because
|
||||
> 27,000 ALL had gone off-book; **every renewal since takes the same off-book path** — the
|
||||
> operator collects the next month's fee and moves `validTo`, with no `payment` event. The
|
||||
> recurring-billing design below closes this: a renewal becomes *paying the next billing period*,
|
||||
> a signed `payment`. Until then, a renewal should be taken as a **new sale** (new subscription
|
||||
> row), not an edit.
|
||||
|
||||
### v2 — quantity, plan timeframes (tariff bridge), reserved spots (built 2026-06-20)
|
||||
|
||||
Three enhancements driven by real scenarios (migration `0011`):
|
||||
@@ -356,6 +365,106 @@ Intended behaviour (to design + build later):
|
||||
> **Explicitly postponed.** For now this is documentation only — no schema, no enforcement. A
|
||||
> subscription is valid whenever it is active and within `validFrom`/`validTo`, all day.
|
||||
|
||||
## Recurring billing — prepaid / postpaid, calendar or anniversary — DESIGN 2026-09-08
|
||||
|
||||
**Design only, nothing built.** Captured from a design conversation with the user (2026-09-08):
|
||||
"a subscriber should prepay or postpay every month, on the 1st or on the day the subscription
|
||||
began; a subscription fixed by a daily tariff, e.g. 300 ALL/day; for prepaid, a notice that a
|
||||
subscription is about to expire so the owner/operator warns the subscriber to pay or lose access."
|
||||
The financial side of this grew into its own page — the [[party-ledger]] — because a postpaid
|
||||
subscriber is a *debtor*, and the site has other debtors (hotels, fleets) and creditors
|
||||
(suppliers). This section is the subscription-shaped part.
|
||||
|
||||
### What is wrong with the one-window model
|
||||
|
||||
A subscription today is **one coverage window** (`validFrom`/`validTo`) sold once: a hotel model.
|
||||
There is no recurring agreement, no due date, no grace, no unpaid balance; prepaid vs postpaid is
|
||||
not expressible, and calendar-anchored billing can only be faked with hand-picked dates. And
|
||||
renewal is off-book (callout above).
|
||||
|
||||
### Split the one row into three concepts
|
||||
|
||||
**1. Plan** — the catalog and versioning stay; a plan version gains a **billing rule**:
|
||||
|
||||
```
|
||||
billing: {
|
||||
mode: "prepaid" | "postpaid",
|
||||
cycle: "day" | "week" | "month", // how often a period is billed
|
||||
anchor: "calendar" | "start", // the 1st of the month, or the sale's anniversary
|
||||
graceDays: number, // access continues this long past due
|
||||
noticeDays: number // "about to expire" window
|
||||
}
|
||||
```
|
||||
|
||||
Recurring plans are **priced per day** (`period: "day"`): a calendar month costs
|
||||
`daysInMonth × 300 ALL`, a partial first month is simply the days left, and **calendar and
|
||||
anniversary anchoring share one formula** (proration falls out for free). Fixed-price monthly
|
||||
plans (`period: "month"`) stay for sites that want a flat number. The hotel "N nights" sale is
|
||||
unchanged (a `"day"` plan over a span, no billing rule).
|
||||
|
||||
**2. Agreement** — the `subscriptions` row: holder, credentials, cars, `validFrom`; for a
|
||||
recurring plan **no `validTo`** (open-ended, ends by revoke/suspend). Fixed spans keep `validTo`.
|
||||
The holder is (or is linked to) a **party** ([[party-ledger]]) — the payer, which for a hotel is
|
||||
the hotel, not the guest.
|
||||
|
||||
**3. Billing periods** — one row per cycle, and each is a **`charge`** on the party ledger:
|
||||
|
||||
```
|
||||
subscription_periods id, subscriptionId, periodFrom, periodTo,
|
||||
amountMinor (from the plan version), currency, dueAt,
|
||||
status {due|paid|overdue|waived}, chargeEventId, paymentEventId?
|
||||
```
|
||||
|
||||
- **Paying a period** = the existing signed **`payment`** with `subscriptionSale: true` plus the
|
||||
period/charge reference — drawer and Z-report keep working with no new summing
|
||||
(§Collecting the fee). **Renewal is just paying the next period.** This closes the off-book hole.
|
||||
- **Waiving** a period is a signed **$0 payment with a reason** — the same rule the Car Wash uses
|
||||
for a comp ([[venue-modules]]: a comp never opens the barrier, sign the $0 payment) — or a
|
||||
`write_off` on the party ledger; admin-gated either way.
|
||||
- The next period is **generated ahead** (prepaid: before the current one ends, so it can be paid
|
||||
early; postpaid: at period end, due `dueAt`), by a daily tick or lazily on read.
|
||||
|
||||
### The gate asks one function
|
||||
|
||||
The entry flow stops reading `validTo` for recurring plans and asks
|
||||
`subscriptionAccess(sub, periods, now) → { ok, reason, accessUntil, daysLeft }`:
|
||||
|
||||
- **prepaid** — allowed while `now ≤ paidThrough + graceDays` (the next period must be paid
|
||||
before it starts, plus grace);
|
||||
- **postpaid** — allowed while no period is unpaid past `dueAt + graceDays`;
|
||||
- both collapse to one derived **`accessUntil`** and **`daysLeft`** per subscriber (never stored).
|
||||
|
||||
This also answers the long-open **lapsed-mid-stay** question for recurring subs: a period ending
|
||||
while a car is parked falls into **grace**, so nobody is trapped; only a subscriber still parked
|
||||
past grace becomes a transient at exit (the tariff-bridge machinery above already prices that).
|
||||
Revoked/suspended behaviour is unchanged.
|
||||
|
||||
### "About to expire" — derived, not stored
|
||||
|
||||
One endpoint (e.g. `GET /api/subscriptions/attention`) lists subscribers whose `accessUntil` falls
|
||||
within the plan's `noticeDays`, those in grace, and those overdue. Surfaced in three places:
|
||||
|
||||
1. a **counter on the booth console** ([[booth-console]]);
|
||||
2. a **badge in the subscriber list**;
|
||||
3. a **line in the live feed when such a subscriber scans in** — "expires in 3 days" at the moment
|
||||
the person is at the gate (a slip can print, best-effort like the window-charge notice).
|
||||
|
||||
Contacting the subscriber stays with the operator/owner by phone (`contact` field). SMS/email
|
||||
is off-appliance ([[cloud-service-saas]]) — a separate decision.
|
||||
|
||||
### Not built, deliberately
|
||||
|
||||
Automatic card charging, invoices, dunning, auto-suspension without grace. **The operator still
|
||||
never types a price.**
|
||||
|
||||
### Build order (after [[party-ledger]] step 1)
|
||||
|
||||
1. Billing rule on the plan version + `subscription_periods` (migration); period generation.
|
||||
2. Pay-period route (signed `payment` + charge reference) and the `subscriptionAccess` gate
|
||||
function in `subscription-flow.ts`; `PUT` stops moving `validTo` on recurring subs.
|
||||
3. Attention endpoint + the three UI surfaces.
|
||||
4. Wiki + [[booth-console]] docs.
|
||||
|
||||
## Data model (as-built 2026-06-18)
|
||||
|
||||
Tables (mutable master data; every *use* still produces a signed `vehicle_entry`/`vehicle_exit`):
|
||||
@@ -420,10 +529,15 @@ subscription** (card/QR credential, or a bound plate) — otherwise to the trans
|
||||
1. **Reader hardware** — confirm the RF reader and QR/optical reader models (procurement; [[bom]],
|
||||
[[open-questions]]).
|
||||
2. **Lapsed-mid-stay & revoked** policy (fall back to transient [[tariff]] vs. refuse) — confirm.
|
||||
3. ~~**Subscription-fee collection**~~ — **RESOLVED + BUILT 2026-06-20.** Selling a priced
|
||||
subscription appends a signed `payment` (`subscriptionSale` flag, `priceMinor × months`,
|
||||
operator-chosen tender) that folds into the drawer/Z-report. Remaining sub-question: should a sale
|
||||
be **hard-blocked without an open shift** (it isn't today — it warns instead)? See "Collecting the
|
||||
fee".
|
||||
3. ~~**Subscription-fee collection**~~ — **RESOLVED + BUILT 2026-06-20** for the *first* sale
|
||||
(signed `payment`, `subscriptionSale` flag, operator-chosen tender, folds into the
|
||||
drawer/Z-report). **REOPENED 2026-09-08 for RENEWALS**: a renewal is a `PUT` that appends
|
||||
nothing (see the callout under "Multi-month"). Closed by the recurring-billing design (a renewal
|
||||
= paying the next period). Remaining sub-question: should a sale be **hard-blocked without an
|
||||
open shift** (it isn't today — it warns instead)?
|
||||
4. **Time-of-day access windows** (overnight subscribers) — design + build; boundary-case policy
|
||||
above (see the design note).
|
||||
5. **Recurring billing** (prepaid/postpaid, calendar/anniversary anchor, grace, expiry notice) —
|
||||
**designed 2026-09-08, not built**; see §Recurring billing and [[party-ledger]]. To refine: is
|
||||
the next period generated by a daily tick or lazily; does a waived period sign a $0 `payment` or
|
||||
a `write_off` (pick one); whether `noticeDays` is per plan or per site.
|
||||
|
||||
+6
-2
@@ -46,6 +46,7 @@ Counts: 4 sources · 19 entities · 47 concepts · 8 decision records.
|
||||
- [[dingtian-relay]] — ✅ CHOSEN access controller; decoupled inputs solve the button blocker (driver verified on hardware); spare relays drive aux outputs (`setAux`).
|
||||
- [[hikvision-radar]] — vehicle-presence radar on a Dingtian input; the entry presence gate (per-input active-level caveat).
|
||||
- [[rongta-printer]] — ✅ CHOSEN 80mm thermal printer; ESC/POS over raw TCP 9100 (or local USB, see [[printer-usb-transport]]); driver written, one unit reachable at 10.0.10.6.
|
||||
- [[k200l-printer]] — the park-buzi printer identified (2026-09-09): Xprinter/ICS K200L, USB id 1fc9:2016 "POS-80", J-Speed LAN board at 192.168.123.100 (DHCP off, web config on :80); status page `/prt_status.htm` (Rongta's rows, headerless HTTP) → own `k200l` driver with raw-socket fetch; over USB reachability only, so cover-open never showed at park-buzi.
|
||||
- [[bom]] — reference bill of materials (barrier, loops, controller, readers, payment, host, network).
|
||||
|
||||
## Concepts — foundational forces
|
||||
@@ -65,6 +66,7 @@ Counts: 4 sources · 19 entities · 47 concepts · 8 decision records.
|
||||
- [[device-adapter-pattern]] — business logic talks to interfaces; swap hardware → new adapter.
|
||||
- [[device-registry]] — catalog of selectable drivers per category (admin-configurable).
|
||||
- [[first-run-setup]] — admin adds controllers + binds readers/cameras to relays from the catalog at install.
|
||||
- [[site-device-installation]] — FIELD RUNBOOK (2026-09-09): per device — factory address + credentials, the tool needed, what the wizard configures itself vs what is done by hand on the device, known traps; address plan, order of work on site, gaps to fill. Dingtian relay, DT-008 readers, Hikvision camera, radar, K200L / Rongta / Cashino printers.
|
||||
- [[device-input-flow]] — button → device push → backend decides → relay; backend is source of truth.
|
||||
- [[device-discovery]] — optional driver capability to scan the LAN (no current driver uses it; UHPPOTE was the example).
|
||||
- [[barrier-not-a-door]] — never timed-close a barrier; safety lives in barrier firmware.
|
||||
@@ -102,7 +104,7 @@ Counts: 4 sources · 19 entities · 47 concepts · 8 decision records.
|
||||
- [[site-metadata]] — optional park identity (name, operator, VAT, address, contact) in site_config; feeds the ticket header.
|
||||
- [[valet-overcapacity]] — "full" is soft: operator may valet-accept over capacity (keys handed over, custody). Manned, deferred.
|
||||
- [[validation-discounts]] — BUILT (2026-07-13): in-park merchant (bar/lavazh) users scan-and-validate on their device (signed event, program↔user binding); booth settles NET + prints gross/discount/net; comp/time-credit/fixed/percent, caps, /setup/site panel, /validate screen.
|
||||
- [[validation-sponsorship]] — design: sponsor accounts + postpaid B2B (customers park free, business billed monthly); not a permit.
|
||||
- [[validation-sponsorship]] — design: postpaid B2B sponsorship (customers park free, business billed monthly); its sponsor-account sketch is superseded by [[party-ledger]].
|
||||
- [[reporting-analytics]] — revenue/occupancy/stay reports + plate-search, all projections over the signed log.
|
||||
- [[clock-integrity]] — fees depend on the host clock; detect/flag backdating on an offline box.
|
||||
- [[ticket-encoding]] — transient ticket id (11-digit numeric + Luhn) as Code128; printed at entry, scanned at pay station + exit; barcode geometry must fit paper width (KP-300H overflow); plate-as-ticket alt.
|
||||
@@ -110,7 +112,7 @@ Counts: 4 sources · 19 entities · 47 concepts · 8 decision records.
|
||||
- [[device-events]] — unsigned hardware telemetry (relay/printer/camera/reader/input); separate from the signed ledger.
|
||||
- [[app-logs]] — the third stream: diagnostic logs (backend warn+ pino sink + frontend errors) → app_logs; log:read viewer; pruned by age+row cap.
|
||||
- [[soft-delete]] — BUILT: accidental admin deletes of master data (users/roles/subs/plans/tariffs) are soft (deleted_at) + recoverable from a recycle bin; auto-purge after N days; signed ledger out of scope.
|
||||
- [[subscription]] — recurring plan (e.g. 10,000 ALL/month); RF/QR or plate identity, car-count + max-concurrent, host-in-loop; short-circuits payment. (Renamed from "permit"; time-of-day windows noted, deferred.)
|
||||
- [[subscription]] — recurring plan (e.g. 10,000 ALL/month); RF/QR or plate identity, car-count + max-concurrent, host-in-loop; short-circuits payment. Plan catalog + tariff bridge built. 2026-09-08: **renewal found off-book**; recurring billing (prepaid/postpaid, calendar/anniversary, grace, expiry notice, billing periods as ledger charges) designed, not built.
|
||||
- [[opencv-anpr-service]] — host-side vision microservice: ANPR (plate identity) + vehicle verification (anti-plate-spoofing witness); fast-alpr (MIT, YOLOv9+CCT/ONNX) the evaluated recognizer baseline.
|
||||
- [[lane-presence-and-anpr-entry]] — camera vehicle detection → (BUILT) advisory lane busy/free booth lights + (BUILT) the ANPR "bridge" (`anpr-entry.ts`): a subscriber's plate read at the lane admits them via the existing gated subscription flow (match-before-emit; subscriber-only). Measured camera limits; rejected the queue-tracking/livestream ideas.
|
||||
- [[vision-service-hardening]] — fix/hardening backlog for `apps/vision/` (2026-07-02 reviews): DoS (body-cap, pixel-bomb, event-loop-blocking inference), unauthenticated + operator-writable model weights, `0.0.0.0` default bind, + correctness/hygiene items. Not yet fixed — the to-do list.
|
||||
@@ -133,9 +135,11 @@ Counts: 4 sources · 19 entities · 47 concepts · 8 decision records.
|
||||
- [[dingtian-vs-mqtt]] — transport choice: direct HTTP/UDP now, MQTT parked until multi-lane scale.
|
||||
- [[session-model]] — business layer start: session = projection; transient-first; pay-on-foot. New event types.
|
||||
- [[vision-service]] — build a host-side ANPR + vehicle-verification service; replaces edge-LPR; scoped AGPL exception.
|
||||
- [[bodytype-classifier-training]] — phase B (SUV vs sedan): `apps/trainer` trains on the collector host (time split, floor, features/finetune modes, feature cache) → report → publish to the Gitea generic package → `models/bodytype.version` pin bakes it into the vision image → TAG bump; sidecar = the preprocessing contract; CPU-only torch on the Xeon E3-1225 v5, Quadro FX 3800 unusable, cloud GPU rejected. Built 2026-09-07; first run waits for labels.
|
||||
- [[vision-service-packaging]] — the vision service lives in this monorepo (apps/vision/), separate process, wired into Turbo via a package.json shim; uv-managed Python.
|
||||
- [[event-streams-split]] — split the signed business ledger (ledger_events) from unsigned device telemetry (device_events).
|
||||
- [[desktop-shell-tauri]] — ✅ Tauri v2 chosen over Electron for the desktop kiosk shell; thin wrapper, server keeps all logic. Best case Ubuntu 26.04 LTS (resolves WebKitGTK); worst case Windows+WSL → kiosk browser, no native shell. Auto-updater mirrors signed releases to public `mca/public_releases` (source repo is private — field appliances have no Gitea creds).
|
||||
- [[party-ledger]] — 🟡 DESIGN (2026-09-08, not built): counterparty sub-ledger for who-owes-whom across modules — parties + signed `charge` / settlement / `write_off` events, balance derived never stored, aging + statements + CSV; lands postpaid subscriptions, hotel guest-nights, fleet washes on account, supplier/utility bills. Sub-ledger only: no bookkeeping, statements are not fiscal invoices, parties per appliance.
|
||||
- [[venue-modules]] — 🟡 OPEN: optional per-site modules (Car Wash, Bar/Restaurant) with Parking as a peer module on a venue POS/audit core; manifest registry, entitled ∩ activated enablement (vendor env + site-admin config), validation kept for the Bar (Lavazh station retires with Car Wash), name stays parking-system, vision vehicle-category as an advisory anomaly flag.
|
||||
- [[container-deployment]] — Docker images for the non-desktop apps: parking-server (Fastify API + bundled SPA via @fastify/static) + parking-vision (Python/uv ANPR); branch+SHA tags, per-env compose, Gitea registry, build-images.yml CI; pnpm deploy (not prune) for native better-sqlite3; migrate-at-boot.
|
||||
- [[fleet-deployment-komodo]] — fleet control plane: Komodo Periphery on each booth, driven by Komodo Core over a NetBird mesh, running the same compose files. Deploys manual + pinned to dev-<sha> (no webhook); secrets Komodo-managed per-booth+unique; booth.sh demoted to break-glass. Threat-model caveats: Periphery is a root agent (mesh-bound only), EVENT_SIGNING_KEY-in-Core is a fraud-root blast radius until ATECC608 signs. komodo/ is infra-as-code.
|
||||
|
||||
+189
@@ -3133,3 +3133,192 @@ in-process) + `enqueueEntry()` (crop + camera class, no order/operator/category)
|
||||
kind column, operator agreement is wash-only. Setup line shows "1 in N entries sampled". Also:
|
||||
Setup → Car wash is a two-column grid (the master-data card was squeezed at max-w-2xl). Tests
|
||||
on both sides. Updated [[vision-review-outbox]].
|
||||
|
||||
## [2026-09-07] decide | Phase B training path + hardware — recorded, not built
|
||||
User asked "now what about the training" and then "let's talk hardware". Recorded on the new
|
||||
[[bodytype-classifier-training]]: the five-step loop (train on the collector host → evaluate with a
|
||||
floor → publish weights to the registry → bake into the vision image → TAG bump; a booth gets a
|
||||
model the way it gets code, never a runtime fetch); ~200 reviewed crops per class before the first
|
||||
run; the Quadro FX 3800 is unusable (cc 1.3), the HD P530 irrelevant, the Xeon E3-1225 v5 is enough
|
||||
(feature-extraction head in minutes, full fine-tune ~1 h); trainer image = CPU-only torch, the
|
||||
compose seam drops the GPU reservation; cloud GPU rejected (crops stay on premises). Linked from
|
||||
[[opencv-anpr-service]], [[vision-review-outbox]], index. User: "No build just yet."
|
||||
|
||||
## [2026-09-07] query | How to seed the admin user on a booth
|
||||
Answered from [[appliance-provisioning]] §7b/§7e (`docker exec … node scripts/seed-admin.mjs`,
|
||||
`FORCE=1` to reset a password). One gap filled: the container-name pattern (`<stack>-server-1`,
|
||||
`park-2-server-1` on park-2), the prompting form, idempotence, and re-seeding after a reset.
|
||||
|
||||
## [2026-09-07] fix | reset-db drift — Car Wash tables and role_jobs were uncategorised
|
||||
User asked for "the command to reset everything in the booth pc". The documented command
|
||||
([[appliance-provisioning]] §7d, `docker exec … reset-db.mjs --all`) would have been refused on
|
||||
park-2: the drift guard found `carwash_*` (six tables) and `role_jobs` outside every category.
|
||||
Categorised (orders + review outbox → financial; prices/categories/services/config → config;
|
||||
role_jobs → users), `--all` verified on a freshly migrated DB. The script ships in the server
|
||||
image — fixed on the booth from the next deployed tag, or by `docker cp` until then. Table
|
||||
rows updated on [[local-dev-workflow]].
|
||||
|
||||
## [2026-09-07] decision | Desktop v0.2.0 — a minor bump, not a patch
|
||||
User: "Do you think we are ready for version 0.2.0? The actual version is 0.1.7." Yes: v0.1.x
|
||||
were all shell fixes; the bundled SPA now carries the module registry, Car Wash + per-till
|
||||
shifts, roles jobs, the vision category and the review outbox (31 commits since v0.1.7).
|
||||
`tauri.conf.json` set to 0.2.0, annotated tag `v0.2.0` created locally; the release gate in
|
||||
apps/desktop/README.md (real bundle, LIVE, a mutation, a frontend log row) stands between the
|
||||
tag and its push. Recorded on [[desktop-shell-tauri]].
|
||||
|
||||
## [2026-09-07] build | Training from the collector UI — the trainer becomes a job service
|
||||
User: the compose-profile trainer is "not so smart" (where is the compose file on a periphery
|
||||
host? why not a button on the collector UI?). Built: `parking-trainer serve` — a stdlib job API
|
||||
(`/health`, `/readiness`, `/versions`, `/versions/<v>/report`, `/jobs`), one job at a time, each
|
||||
job the CLI as a subprocess with state + log persisted under `/out/jobs/`; the collector gained
|
||||
`COLLECTOR_TRAINER_URL` + `/api/training/*` (reviewer-gated proxy, fixed paths, whitelisted
|
||||
knobs, trainer status codes passed through, 503 unconfigured / 502 unreachable) and a Training
|
||||
section on `/review` (readiness table, Train with mode/backbone/floor, live log, versions with
|
||||
Report / Evaluate / Publish, the pin reminder). Compose: `trainer` is a service now
|
||||
(`restart: unless-stopped`, `serve`, read-only data, own `trainer-out` volume, not published);
|
||||
the Docker socket route was rejected (root on the host for a service booths upload to). Tests:
|
||||
trainer 14, collector 7. Pages: [[bodytype-classifier-training]] (loop, running it, operating
|
||||
notes superseded), [[vision-review-outbox]], [[fleet-deployment-komodo]].
|
||||
|
||||
## [2026-09-07] ingest | Trainer deployed as a profile; operating notes
|
||||
User pushed `stage-f7a262a`, bumped the `wash-collector` TAG, redeployed, and asked why only one
|
||||
service runs on art-docker-station. Expected: the trainer is a compose profile, never started or
|
||||
pulled by a deploy; run by hand, exits. Recorded on [[bodytype-classifier-training]] §Operating
|
||||
notes (why the TAG bump still mattered, park-2 needs no bump until a pin, the registry login for
|
||||
the first pull, the order of commands) and [[fleet-deployment-komodo]].
|
||||
|
||||
## [2026-09-07] build | Phase B trainer + the classifier stage on the booth
|
||||
User: "Shall we go and build the trainer for the Xeon?" Built `apps/trainer` (`parking-trainer`:
|
||||
`inspect` / `train` / `evaluate` / `publish`; reads the collector volume read-only, time split,
|
||||
thin classes dropped, damped class weights, `features` mode with an on-disk feature cache and
|
||||
`finetune` mode with light augmentation, CPU-only torch from PyTorch's wheel index, ONNX export
|
||||
checked against the torch model, **no model file below the floor** — exit 3 with the report; exit
|
||||
2 = not enough labels), its image + a `.dockerignore`, and the `trainer` compose profile on the
|
||||
collector stack (CPU, read-only data volume, `TRAINER_OUT`). Vision side: `BodyTypeClassifier` +
|
||||
`RefinedVehicleDetector` (car or a known class only; min-confidence; `detector_class`; missing
|
||||
file = off without an error, broken file = health detail), `models/bodytype.version` pin fetched
|
||||
at build from the Gitea generic package (BuildKit secret; a pin that cannot be fetched fails the
|
||||
build). The sidecar is the preprocessing contract; verified a trainer model gives identical
|
||||
probabilities inside the vision service. CI: trainer synced without the `train` extra, torch tests
|
||||
skip. Tests: trainer 10 (6 in CI mode), vision 18. Pages: [[bodytype-classifier-training]]
|
||||
rewritten as built, [[opencv-anpr-service]] §Phase B, [[vision-review-outbox]],
|
||||
[[vision-service-packaging]], [[fleet-deployment-komodo]], index.
|
||||
|
||||
## [2026-09-07] ingest | Collector live on park-2; secrets shape, DNS vs bind, token format, CI rule
|
||||
Deployed: collector on art-docker-station + park-2 at stage-dbbb051, every entry sampled; review
|
||||
screen filling. Recorded on [[vision-review-outbox]]: one secret per booth referenced by both stacks
|
||||
(the combined-list secret was replaced the same day), URL by Netbird DNS name vs bind by IP, token =
|
||||
`openssl rand -hex 32` (opaque, ≥16, no separators, never shared), the operator hash needs no
|
||||
variable, deploy the collector before a booth that sends a new package kind, the export's formula
|
||||
neutralisation (security review finding), and why every entry is sent. On
|
||||
[[vision-service-packaging]]: CI syncs without the alpr extra — numpy in the dev group, cv2 tests
|
||||
importorskip (three red runs on 2026-09-07).
|
||||
|
||||
## [2026-09-08] decision | Subscription recurring billing + the party ledger (design only)
|
||||
User: "an subscriber should prepay or postpay every month, at the 1st or the day it began; fixed
|
||||
by a daily tariff (300 ALL/day); notify when about to expire … we need a more flexible way." Then:
|
||||
"the financial aspect is too simple" — postpaid agreements, hotels given daily access for guests,
|
||||
Car Wash fleet deals paid per period, the wash's supplier/utility bills; the admin needs to see
|
||||
uncollected dues. Assessed against the code: a subscription is ONE coverage window sold once;
|
||||
**renewal is off-book** (a `PUT` that never re-sells and appends no `payment` — the same hole
|
||||
closed for the first sale on 2026-06-20). Designed, not built: (a) [[subscription]] §Recurring
|
||||
billing — plan billing rule {mode, cycle, anchor, graceDays, noticeDays}, recurring plans priced
|
||||
per day so calendar and anniversary anchoring share one formula, open-ended agreement, a
|
||||
`subscription_periods` table where each period is a ledger charge and a renewal = paying the next
|
||||
period (signed `payment`), one `subscriptionAccess()` gate function (answers lapsed-mid-stay via
|
||||
grace), expiry notice derived not stored (console counter, list badge, feed line at scan-in);
|
||||
(b) new [[party-ledger]] decision page — parties + signed `charge` / settlement (`payment` /
|
||||
`cash_out` / bank) / `write_off`, balance derived never stored (threat model), aging + statements
|
||||
+ CSV, the four cases (subscriptions, hotels, fleet washes as `payAt: "account"`, supplier bills
|
||||
as payables), the line drawn (sub-ledger, not bookkeeping; a statement is not a fiscal invoice;
|
||||
parties per appliance), build order. [[validation-sponsorship]]'s sponsor table marked
|
||||
superseded; [[open-questions]] #17 added, #3 on the subscription page reopened for renewals;
|
||||
index updated. Nothing in code changed.
|
||||
|
||||
## [2026-09-09] ingest | Printer cover-open bug — bench result falsifies re-enumeration; park-lab rejoins the fleet
|
||||
The failing park-buzi printer is on the dev bench: identified as USB `1fc9:2016` "Printer POS-80"
|
||||
(NXP controller, no brand in the descriptor — hence "Generic" in the app). Attached to WSL via
|
||||
usbipd-win 5.3 (busid 8-1). Cover cycled under `dmesg -w` + `lsusb`: NO disconnect, NO
|
||||
re-enumeration — the leading hypothesis (cover cuts the USB board → stale container `/dev/usb`
|
||||
bind-mount) is falsified for this unit; the fault is in how usblp / the open-probe reacts to the
|
||||
printer's error state. Next discriminator = the monitor's offline `detail` text on park-buzi
|
||||
(EBUSY vs open-timeout vs EIO). WSL caveat recorded: Microsoft's 6.6.87 kernel has
|
||||
CONFIG_USB_PRINTER unset — no `/dev/usb/lpN` without a custom kernel, so the user will reproduce
|
||||
on a Linux box instead. `komodo/resources.toml`: `park-lab` stack re-added for that box — copied
|
||||
from park-2 then corrected (the copy carried park-2's review outbox + booth-2 token; removed — the
|
||||
bench must never feed the collector under a booth id); pinned to the booth's stage-<sha>. Pages:
|
||||
[[printer-usb-transport]] (identity, bench result, WSL caveat), [[fleet-deployment-komodo]]
|
||||
(park-lab row).
|
||||
|
||||
## [2026-09-09] ingest | Printer cover-open bug — lab did NOT reproduce; park-buzi closed, evidence pending
|
||||
Same printer unit on `park-lab` (real Linux host, booth image stage-2d9bb15, prod compose, dev DB
|
||||
snapshot with the USB printer as booth-receipt, cards via subscription reprint): paper out → cover
|
||||
→ reload → reprint — no error. Printer, USB transport and compose wiring cleared in isolation;
|
||||
what remains is park-buzi's environment (kernel/USB path/power) or the container's history.
|
||||
park-buzi is shut down (staff shortage). Recorded on [[printer-usb-transport]]: the three
|
||||
commands to run FIRST when the box next powers on (container log transitions, kernel journal,
|
||||
`lsusb -t`), how to read each answer, and two proposed no-booth follow-ups (close the
|
||||
`withTimeout` handle leak; make the monitor self-document consecutive USB offline polls).
|
||||
Dev-DB snapshot procedure for a lab (online backup → reset-db --financial --diagnostics → clear
|
||||
backup fields → VACUUM → copy into the volume with chown) used today; not yet on a wiki page.
|
||||
|
||||
## [2026-09-09] build | K200L printer identified + `k200l` driver with live status; network setup recorded
|
||||
The bottom label says **Model K200L** (Xprinter/ICS XP-K200L family, USB+LAN, ESC/POS); the LAN
|
||||
board ("J-Speed Ethernet WebConfig 1.02") calls it "POS-80", as does the USB descriptor. User
|
||||
configured it from factory 192.168.123.100 (DHCP off, web UI on :80, no auth) to 10.0.10.7 on the
|
||||
lab; added as booth printer on `park-lab` on the generic driver → badge stayed green with the cover
|
||||
open, because the generic driver is reachability-only by design. Found the board's status page:
|
||||
**`/prt_status.htm`**, the Rongta's five rows exactly, but the reply has NO status line/headers
|
||||
(node:http: "Parse Error: Expected HTTP/"; curl: 000/empty) — so a Rongta-path option would not
|
||||
have worked. Per the user ("this is not rongta", "create a new printer"): the Rongta driver is
|
||||
untouched; new **`printer-k200l.ts`** (`k200l`) delegates printing to the generic ESC/POS device
|
||||
and reads the page over a raw socket, tolerant of both reply shapes; mapping mirrors the Rongta
|
||||
(unreachable → offline, page not understood → degraded never ready, faults → degraded named, USB →
|
||||
floor). Tests replay the captured headerless page (10 new, devices suite 76 green); live probe
|
||||
against 10.0.10.7 → ready; bench with cover open showed "cover open, paper out, printer off-line".
|
||||
Consequence for park-buzi: over USB the app never saw cover/paper state at all — the reported
|
||||
"offline" is a probe failure (errno still to be pulled). Pages: new [[k200l-printer]] (names,
|
||||
network setup runbook, board quirks, status page, park-buzi implication), [[rongta-printer]],
|
||||
[[printer-status-monitoring]], [[printer-usb-transport]], [[wsl-dev-networking]] (parking-net
|
||||
pinned to eth1 while the LAN NIC is eth0 — the source-address bug bit again), index.
|
||||
|
||||
## [2026-09-09] query | Field runbook: installing the devices at a site (what to know before the booth)
|
||||
User: "we need a section about installing these devices in the park sites … I didn't know this
|
||||
printer has initial IP 192.168.123.100 and a web interface … also dingtian relays and readers,
|
||||
cashino printers — know beforehand, not struggle on site." New reference page
|
||||
[[site-device-installation]], synthesised from the entity pages + memory notes: the bring list,
|
||||
the site address plan (10.0.10.x convention, every device static), then per device — Dingtian
|
||||
relay (factory 192.168.1.100, admin/admin, what harden() does vs the by-hand IP + UDP2 disable,
|
||||
wiring I1/I2, unauthenticated CGI, session_en brick), DT-008 readers (192.168.1.99, the
|
||||
QRCode_v1_6_5.exe tool: unique IP, server target, Q:/K: prefixes, 6H card format, QR+Code128
|
||||
only, serial binding, re-apply after a reset), Hikvision G3H (192.168.1.64 + activation, SUB
|
||||
stream mandatory, Alarm Server after assign, Vehicle target filter, close the web UI, corrupt-DB
|
||||
factory reset), radar (idle level → activeLow), printers (K200L / Rongta / Cashino table; LAN over
|
||||
USB; "Test" probes, print a card to verify), the on-site order of work, and the gaps still
|
||||
unrecorded (Cashino/Rongta factory addresses, the reader tool screens, camera activation, where
|
||||
the site record lives). Linked from [[appliance-provisioning]] and [[k200l-printer]]; indexed.
|
||||
|
||||
## [2026-09-09] fix | K200L status parser — a fault's "Yes" is wrapped in <FONT color=#ff0000>
|
||||
First live run on park-lab (driver `k200l`, LAN, cover open) showed *degraded — unexpected status
|
||||
page (missing coverOpen, paperEnd, offline)*: precisely the three Yes cells. Captured the raw page
|
||||
with the cover open: the board writes `<FONT color=#ff0000>Yes</FONT>` for a fault and a bare
|
||||
padded `No` otherwise; the parser accepted only tag-free cells. Fix: cell text is read with inner
|
||||
tags stripped (row-anchored regex); tests pin the verbatim markup plus other shapes (devices 79).
|
||||
Live after the fix: degraded "cover open, paper out, printer off-line"; closed → ready. User:
|
||||
"we are good using the network with this printer." Also: park-lab Periphery was "not loaded"
|
||||
after a reboot — the unit had never been enabled; `systemctl --user enable --now periphery`
|
||||
recorded as a §7a gotcha on [[appliance-provisioning]]. Pages: [[k200l-printer]].
|
||||
|
||||
## [2026-09-16] fix | Collector DB schema migration; trainer handlers answer 500 JSON; the "trainer not reachable" incident
|
||||
User: "Training — trainer not reachable: fetch failed". Read-only look at art-docker-station over
|
||||
SSH: collector stage-2d9bb15 unhealthy (`/health` 500 "no such column: kind"), trainer healthy
|
||||
but `/readiness` tracebacks on the same column; the volume's collector.sqlite (2026-09-07, 0
|
||||
items, original columns) predates `kind` — CREATE TABLE IF NOT EXISTS never migrates an existing
|
||||
table. No `/ingest` request in the container's 9-day log at all (park-2 either not sending or not
|
||||
reaching the host — to check on the booth). Built: `CollectorDb.#migrate()` (PRAGMA table_info vs
|
||||
the list of columns added since the first deploy → ALTER TABLE ADD COLUMN; test replays the old
|
||||
schema: health, ingest, stats, legacy row reads back with defaults); trainer `Handler._guarded`
|
||||
(any exception → 500 JSON naming it; test: readiness on an old-schema DB → 500 "no such column:
|
||||
kind", /health still 200); the collector's training proxy includes the trainer's error text. Pages:
|
||||
[[vision-review-outbox]] (incident + rule), [[bodytype-classifier-training]] (what the message
|
||||
means). Deploy: nothing manual — the new collector migrates on start.
|
||||
|
||||
Reference in New Issue
Block a user