feat(collector): review collector skeleton — apps/collector, its own Komodo stack on the reviewer's host
The far end of the Car Wash review outbox (wiki/concepts/vision-review-outbox.md): a small Fastify + SQLite service in the monorepo (shares the payload contract and the class vocabulary via @parking/shared), delivered to art-docker-station by its own stack so nothing booth-side lands there and nothing of it on a booth. - POST /ingest: bearer token per booth (constant-time), X-Booth-Id must match, multipart meta + JPEG (magic checked, 2 MB cap), meta validated against the contract, idempotent on the item id; crop stored at crops/<booth>/<item>.jpg on the volume + one items row. - /review + /api/*: the reviewer's screen served by the process (Basic auth, one login): one pending crop at a time, operator's pick and camera's pick beside it, one button/key per vocabulary class + unusable + skip; stats per booth and per hashed operator (agree / disagree / unusable — disagree = the reviewer's class is outside the operator's category). - GET /export/labels.csv: reviewed usable rows for training; formula-leading cells are neutralised (booth-supplied names). Crops stay on the volume for the trainer on the host. - Booth payload now carries operatorCategory.classes so the comparison needs no site setup. - Delivery: apps/collector/Dockerfile (monorepo context), docker-compose.collector.yml (bind to the overlay IP; commented `trainer` profile seam for the GPU), a third build step in build-images.yml, a `wash-collector` stack in komodo/resources.toml with one secret per booth referenced from both the collector's token list and the booth's own stack (park-2 lines templated, commented, DNS name for the URL). - Tests: app.test.ts (ingest ok/dup/refusals, review + stats + export, config). Image built and smoke-tested locally (health, ingest, duplicate, auth, verdict, export). Claude-Session: https://claude.ai/code/session_01FWncR69HgGPuei1dLrW3cU
This commit is contained in:
@@ -0,0 +1,179 @@
|
||||
import Database from "better-sqlite3";
|
||||
import type { VehicleClass } from "@parking/shared";
|
||||
|
||||
// One table. Each row is one booth decision: what the camera saw, what the operator
|
||||
// chose, and (once reviewed) what a trusted person says the vehicle is. The crop itself
|
||||
// lives on disk beside the DB (crops/<booth>/<item>.jpg) so the trainer on the same host
|
||||
// reads it straight off the volume.
|
||||
|
||||
export interface ItemRow {
|
||||
id: string;
|
||||
booth: string;
|
||||
orderRef: string;
|
||||
at: string;
|
||||
operatorRef: string;
|
||||
operatorCategoryId: string;
|
||||
operatorCategoryName: string;
|
||||
/** The vision classes the operator's category covers at that site (its mapping) — what
|
||||
* lets a reviewer's CLASS be compared with an operator's CATEGORY. JSON array. */
|
||||
operatorClasses: string;
|
||||
service: string;
|
||||
visionClass: string;
|
||||
visionConfidence: number;
|
||||
visionCategoryId: string | null;
|
||||
downgraded: number;
|
||||
imageWidth: number;
|
||||
imageHeight: number;
|
||||
plateBlurred: number;
|
||||
imagePath: string;
|
||||
receivedAt: string;
|
||||
reviewLabel: string | null; // a VehicleClass, or "unusable"
|
||||
reviewedAt: string | null;
|
||||
reviewer: string | null;
|
||||
}
|
||||
|
||||
export type ReviewVerdict = VehicleClass | "unusable";
|
||||
|
||||
export class CollectorDb {
|
||||
readonly #db: Database.Database;
|
||||
|
||||
constructor(file: string) {
|
||||
this.#db = new Database(file);
|
||||
this.#db.pragma("journal_mode = WAL");
|
||||
this.#db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS items (
|
||||
id TEXT PRIMARY KEY,
|
||||
booth TEXT NOT NULL,
|
||||
order_ref TEXT NOT NULL,
|
||||
at TEXT NOT NULL,
|
||||
operator_ref TEXT NOT NULL,
|
||||
operator_category_id TEXT NOT NULL,
|
||||
operator_category_name TEXT NOT NULL,
|
||||
operator_classes TEXT NOT NULL DEFAULT '[]',
|
||||
service TEXT NOT NULL,
|
||||
vision_class TEXT NOT NULL,
|
||||
vision_confidence REAL NOT NULL,
|
||||
vision_category_id TEXT,
|
||||
downgraded INTEGER NOT NULL DEFAULT 0,
|
||||
image_width INTEGER NOT NULL,
|
||||
image_height INTEGER NOT NULL,
|
||||
plate_blurred INTEGER NOT NULL,
|
||||
image_path TEXT NOT NULL,
|
||||
received_at TEXT NOT NULL,
|
||||
review_label TEXT,
|
||||
reviewed_at TEXT,
|
||||
reviewer TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS items_pending ON items (reviewed_at, received_at);
|
||||
CREATE INDEX IF NOT EXISTS items_booth ON items (booth, received_at);
|
||||
`);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.#db.close();
|
||||
}
|
||||
|
||||
static #map(r: Record<string, unknown>): ItemRow {
|
||||
return {
|
||||
id: r.id as string,
|
||||
booth: r.booth as string,
|
||||
orderRef: r.order_ref as string,
|
||||
at: r.at as string,
|
||||
operatorRef: r.operator_ref as string,
|
||||
operatorCategoryId: r.operator_category_id as string,
|
||||
operatorCategoryName: r.operator_category_name as string,
|
||||
operatorClasses: r.operator_classes as string,
|
||||
service: r.service as string,
|
||||
visionClass: r.vision_class as string,
|
||||
visionConfidence: r.vision_confidence as number,
|
||||
visionCategoryId: (r.vision_category_id as string | null) ?? null,
|
||||
downgraded: r.downgraded as number,
|
||||
imageWidth: r.image_width as number,
|
||||
imageHeight: r.image_height as number,
|
||||
plateBlurred: r.plate_blurred as number,
|
||||
imagePath: r.image_path as string,
|
||||
receivedAt: r.received_at as string,
|
||||
reviewLabel: (r.review_label as string | null) ?? null,
|
||||
reviewedAt: (r.reviewed_at as string | null) ?? null,
|
||||
reviewer: (r.reviewer as string | null) ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
get(id: string): ItemRow | null {
|
||||
const r = this.#db.prepare("SELECT * FROM items WHERE id = ?").get(id) as Record<string, unknown> | undefined;
|
||||
return r ? CollectorDb.#map(r) : null;
|
||||
}
|
||||
|
||||
insert(row: Omit<ItemRow, "reviewLabel" | "reviewedAt" | "reviewer">): void {
|
||||
this.#db
|
||||
.prepare(
|
||||
`INSERT INTO items (id, booth, order_ref, at, operator_ref, operator_category_id, operator_category_name,
|
||||
operator_classes, service, vision_class, vision_confidence, vision_category_id, downgraded,
|
||||
image_width, image_height, plate_blurred, image_path, received_at)
|
||||
VALUES (@id, @booth, @orderRef, @at, @operatorRef, @operatorCategoryId, @operatorCategoryName,
|
||||
@operatorClasses, @service, @visionClass, @visionConfidence, @visionCategoryId, @downgraded,
|
||||
@imageWidth, @imageHeight, @plateBlurred, @imagePath, @receivedAt)`,
|
||||
)
|
||||
.run(row);
|
||||
}
|
||||
|
||||
list(status: "pending" | "reviewed", limit: number, booth?: string): ItemRow[] {
|
||||
const where = [status === "pending" ? "reviewed_at IS NULL" : "reviewed_at IS NOT NULL"];
|
||||
const params: unknown[] = [];
|
||||
if (booth) {
|
||||
where.push("booth = ?");
|
||||
params.push(booth);
|
||||
}
|
||||
const order = status === "pending" ? "received_at ASC" : "reviewed_at DESC";
|
||||
const rows = this.#db
|
||||
.prepare(`SELECT * FROM items WHERE ${where.join(" AND ")} ORDER BY ${order} LIMIT ?`)
|
||||
.all(...params, limit) as Record<string, unknown>[];
|
||||
return rows.map((r) => CollectorDb.#map(r));
|
||||
}
|
||||
|
||||
review(id: string, label: ReviewVerdict, reviewer: string): ItemRow | null {
|
||||
this.#db
|
||||
.prepare("UPDATE items SET review_label = ?, reviewed_at = ?, reviewer = ? WHERE id = ?")
|
||||
.run(label, new Date().toISOString(), reviewer, id);
|
||||
return this.get(id);
|
||||
}
|
||||
|
||||
/** Per booth: received / pending / reviewed. Per operator (booth + hash): how often the
|
||||
* reviewer's class fell inside the operator's chosen category (agree) or outside
|
||||
* (disagree) — the honest-mistake / fraud rate the outbox exists for. */
|
||||
stats(): {
|
||||
booths: { booth: string; received: number; pending: number; reviewed: number }[];
|
||||
operators: { booth: string; operatorRef: string; reviewed: number; agree: number; disagree: number; unusable: number }[];
|
||||
} {
|
||||
const booths = this.#db
|
||||
.prepare(
|
||||
`SELECT booth, COUNT(*) AS received,
|
||||
SUM(CASE WHEN reviewed_at IS NULL THEN 1 ELSE 0 END) AS pending,
|
||||
SUM(CASE WHEN reviewed_at IS NOT NULL THEN 1 ELSE 0 END) AS reviewed
|
||||
FROM items GROUP BY booth ORDER BY booth`,
|
||||
)
|
||||
.all() as { booth: string; received: number; pending: number; reviewed: number }[];
|
||||
const reviewed = this.#db
|
||||
.prepare("SELECT booth, operator_ref, operator_classes, review_label FROM items WHERE reviewed_at IS NOT NULL")
|
||||
.all() as { booth: string; operator_ref: string; operator_classes: string; review_label: string }[];
|
||||
const ops = new Map<string, { booth: string; operatorRef: string; reviewed: number; agree: number; disagree: number; unusable: number }>();
|
||||
for (const r of reviewed) {
|
||||
const key = `${r.booth} ${r.operator_ref}`;
|
||||
let o = ops.get(key);
|
||||
if (!o) ops.set(key, (o = { booth: r.booth, operatorRef: r.operator_ref, reviewed: 0, agree: 0, disagree: 0, unusable: 0 }));
|
||||
o.reviewed += 1;
|
||||
if (r.review_label === "unusable") o.unusable += 1;
|
||||
else if ((JSON.parse(r.operator_classes) as string[]).includes(r.review_label)) o.agree += 1;
|
||||
else o.disagree += 1;
|
||||
}
|
||||
return { booths, operators: [...ops.values()].sort((a, b) => b.disagree - a.disagree) };
|
||||
}
|
||||
|
||||
/** Reviewed, usable rows — the training set. */
|
||||
labelled(): ItemRow[] {
|
||||
const rows = this.#db
|
||||
.prepare("SELECT * FROM items WHERE reviewed_at IS NOT NULL AND review_label != 'unusable' ORDER BY reviewed_at")
|
||||
.all() as Record<string, unknown>[];
|
||||
return rows.map((r) => CollectorDb.#map(r));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user