From f7a262ac9a7f075a7d76db37d7723ebddd6a6706 Mon Sep 17 00:00:00 2001 From: Julian Cuni Date: Mon, 7 Sep 2026 11:14:50 +0200 Subject: [PATCH] =?UTF-8?q?feat(trainer):=20phase-B=20body-type=20classifi?= =?UTF-8?q?er=20=E2=80=94=20trainer=20job=20on=20the=20collector=20host=20?= =?UTF-8?q?+=20the=20classifier=20stage=20on=20the=20booth?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apps/trainer (parking-trainer): inspect / train / evaluate / publish. Reads the wash collector's SQLite + crops read-only off its volume; time split (validation = newest slice); thin classes dropped; damped class weights; `features` mode (frozen ImageNet backbone, on-disk feature cache, seconds to retrain) and `finetune` mode (light augmentation). CPU-only torch from PyTorch's wheel index. ONNX export checked against the torch model; NO model file below the validation floor (exit 3, report still written); exit 2 = not enough labels. `evaluate` scores a shipped model on labels reviewed after training + the unlabelled pile; `publish` PUTs a version folder to a Gitea generic package. Light core deps; the `train` extra is heavy — CI syncs without it, torch tests skip. apps/vision: BodyTypeClassifier (bodytype.onnx + sidecar = the preprocessing contract: crop margin, input size, RGB 0-255, normalisation inside the graph) and RefinedVehicleDetector over YOLOX — refines only `car` or a class the classifier trained on, min-confidence, `detector_class` on the result; path set but no file = phase B off without an error; a broken file is a health detail. models/bodytype.version (tracked, empty) pins the published version the Dockerfile fetches at build (BuildKit secret; a pin that cannot be fetched fails the build). Verified: a trainer model gives identical probabilities inside the vision service; both images built and smoke-tested. Delivery: parking-trainer image in build-images.yml, the `trainer` compose profile on the collector stack (CPU, read-only data, TRAINER_OUT), commented TRAINER_OUT/PUBLISH_TOKEN in the wash-collector stack, .dockerignore for both Python contexts, trainer deps synced in CI. Wiki: bodytype-classifier-training rewritten as built (+ one fleet model not per site, secrets/access, where the crops live), opencv-anpr-service §Phase B, vision-review-outbox, vision-service-packaging, fleet-deployment-komodo, index, log. Claude-Session: https://claude.ai/code/session_01FWncR69HgGPuei1dLrW3cU --- .gitea/workflows/build-images.yml | 25 +- .gitea/workflows/ci.yml | 5 + apps/trainer/.dockerignore | 7 + apps/trainer/.gitignore | 12 + apps/trainer/.python-version | 1 + apps/trainer/Dockerfile | 43 + apps/trainer/README.md | 36 + apps/trainer/package.json | 13 + apps/trainer/pyproject.toml | 73 + apps/trainer/tests/conftest.py | 102 ++ apps/trainer/tests/test_data.py | 98 ++ apps/trainer/tests/test_train.py | 155 ++ apps/trainer/trainer/__init__.py | 7 + apps/trainer/trainer/cli.py | 388 +++++ apps/trainer/trainer/data.py | 179 ++ apps/trainer/trainer/infer.py | 51 + apps/trainer/trainer/model.py | 305 ++++ apps/trainer/trainer/preprocess.py | 88 + apps/trainer/trainer/report.py | 152 ++ apps/trainer/turbo.json | 9 + apps/trainer/uv.lock | 1444 +++++++++++++++++ apps/vision/.dockerignore | 9 + apps/vision/.env.example | 8 + apps/vision/.gitignore | 3 +- apps/vision/Dockerfile | 21 +- apps/vision/models/bodytype.version | 0 apps/vision/tests/test_vehicle.py | 132 ++ apps/vision/vision_service/recognizer.py | 19 +- apps/vision/vision_service/schemas.py | 3 + apps/vision/vision_service/settings.py | 8 + apps/vision/vision_service/vehicle.py | 170 ++ docker-compose.collector.yml | 37 +- komodo/resources.toml | 5 + pnpm-lock.yaml | 2 + wiki/concepts/vision-review-outbox.md | 16 +- .../decisions/bodytype-classifier-training.md | 184 ++- wiki/decisions/fleet-deployment-komodo.md | 5 + wiki/decisions/vision-service-packaging.md | 8 + wiki/entities/opencv-anpr-service.md | 31 +- wiki/index.md | 2 +- wiki/log.md | 17 + 41 files changed, 3797 insertions(+), 76 deletions(-) create mode 100644 apps/trainer/.dockerignore create mode 100644 apps/trainer/.gitignore create mode 100644 apps/trainer/.python-version create mode 100644 apps/trainer/Dockerfile create mode 100644 apps/trainer/README.md create mode 100644 apps/trainer/package.json create mode 100644 apps/trainer/pyproject.toml create mode 100644 apps/trainer/tests/conftest.py create mode 100644 apps/trainer/tests/test_data.py create mode 100644 apps/trainer/tests/test_train.py create mode 100644 apps/trainer/trainer/__init__.py create mode 100644 apps/trainer/trainer/cli.py create mode 100644 apps/trainer/trainer/data.py create mode 100644 apps/trainer/trainer/infer.py create mode 100644 apps/trainer/trainer/model.py create mode 100644 apps/trainer/trainer/preprocess.py create mode 100644 apps/trainer/trainer/report.py create mode 100644 apps/trainer/turbo.json create mode 100644 apps/trainer/uv.lock create mode 100644 apps/vision/.dockerignore create mode 100644 apps/vision/models/bodytype.version diff --git a/.gitea/workflows/build-images.yml b/.gitea/workflows/build-images.yml index ab1e6f7..60a0599 100644 --- a/.gitea/workflows/build-images.yml +++ b/.gitea/workflows/build-images.yml @@ -1,6 +1,6 @@ name: Build & push images -# Build the SERVER (API + SPA), COLLECTOR (wash review) and VISION (ANPR) container images and push them to the +# Build the SERVER (API + SPA), COLLECTOR (wash review), VISION (ANPR) and TRAINER (phase-B job) container images and push them to the # house Gitea registry, tagged by BRANCH + short SHA (branch-aware: dev→:dev, stage→:stage, # main→:main). Separate from ci.yml (checks-only) and release.yml (tag-only desktop bundle). # Mirrors the house pattern (cf. trm/processor build.yml). See @@ -14,6 +14,7 @@ on: - 'apps/web/**' - 'apps/vision/**' - 'apps/collector/**' + - 'apps/trainer/**' - 'packages/**' - 'package.json' - 'pnpm-lock.yaml' @@ -61,6 +62,11 @@ jobs: working-directory: apps/vision run: uv sync --frozen + - name: Sync trainer deps + # Light core only — NOT the `train` extra (CPU torch, ~200 MB); the torch tests skip. + working-directory: apps/trainer + run: uv sync --frozen + # Don't publish a broken image — run the same checks as ci.yml first. - name: Build + lint + test (Turbo) run: pnpm turbo run build lint test @@ -119,12 +125,29 @@ jobs: context: apps/vision file: apps/vision/Dockerfile push: true + # The phase-B body-type classifier is fetched from the Gitea generic package registry + # at build when apps/vision/models/bodytype.version pins a version (empty = none). The + # registry user's credentials double as the fetch auth (BuildKit secret, never a layer). + secrets: | + bodytype_auth=${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }} tags: | ${{ env.REGISTRY }}/parking-vision:${{ steps.meta.outputs.branch }} ${{ env.REGISTRY }}/parking-vision:${{ steps.meta.outputs.branch }}-${{ steps.meta.outputs.sha }} cache-from: type=registry,ref=${{ env.REGISTRY }}/parking-vision:buildcache cache-to: type=registry,ref=${{ env.REGISTRY }}/parking-vision:buildcache,mode=max + - name: Build & push TRAINER (phase-B job) + uses: docker/build-push-action@v5 + with: + context: apps/trainer + file: apps/trainer/Dockerfile + push: true + tags: | + ${{ env.REGISTRY }}/parking-trainer:${{ steps.meta.outputs.branch }} + ${{ env.REGISTRY }}/parking-trainer:${{ steps.meta.outputs.branch }}-${{ steps.meta.outputs.sha }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/parking-trainer:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/parking-trainer:buildcache,mode=max + # Optional: trigger a Komodo stack redeploy (cf. trm/processor). Enable by setting the # KOMODO_* secrets; left guarded so it no-ops until the parking stack is wired. - name: Trigger Komodo redeploy diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 16f97c2..8d45f01 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -48,6 +48,11 @@ jobs: working-directory: apps/vision run: uv sync --frozen + - name: Sync trainer deps + # Same rule: light core only, not the `train` extra (CPU torch); torch tests skip. + working-directory: apps/trainer + run: uv sync --frozen + - name: Build + lint (Turbo) # Covers tsc typecheck, vite build, i18n catalog type-parity (a missing sq/en # key fails the build), AND the vision service's ruff lint via uv. diff --git a/apps/trainer/.dockerignore b/apps/trainer/.dockerignore new file mode 100644 index 0000000..c36230c --- /dev/null +++ b/apps/trainer/.dockerignore @@ -0,0 +1,7 @@ +.venv/ +**/__pycache__/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +out/ +.env diff --git a/apps/trainer/.gitignore b/apps/trainer/.gitignore new file mode 100644 index 0000000..20be638 --- /dev/null +++ b/apps/trainer/.gitignore @@ -0,0 +1,12 @@ +# Python +__pycache__/ +*.py[cod] +.venv/ +.mypy_cache/ +.pytest_cache/ +.ruff_cache/ + +# Model weights (fetched at deploy / first run, never committed — can be large + license-scoped) +out/ + +*.onnx diff --git a/apps/trainer/.python-version b/apps/trainer/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/apps/trainer/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/apps/trainer/Dockerfile b/apps/trainer/Dockerfile new file mode 100644 index 0000000..475177e --- /dev/null +++ b/apps/trainer/Dockerfile @@ -0,0 +1,43 @@ +# syntax=docker/dockerfile:1.7 +# Parking TRAINER image: the phase-B body-type classifier job. Build CONTEXT is apps/trainer +# (self-contained Python package). A ONE-OFF JOB on the reviewer's host (art-docker-station), +# never a booth service: it reads the wash collector's volume (collector.sqlite + crops/) +# and writes a versioned model folder. CPU-only PyTorch — the host has no usable GPU and a +# few thousand crops train in minutes/an hour on four Xeon cores. +# See wiki/decisions/bodytype-classifier-training.md. + +FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS base +WORKDIR /app +ENV UV_LINK_MODE=copy \ + UV_COMPILE_BYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libgl1 libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +COPY pyproject.toml uv.lock .python-version ./ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-install-project --no-dev --extra train + +COPY trainer/ ./trainer/ +COPY README.md ./ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev --extra train + +# Pre-warm the ImageNet backbone weights INTO the image so a run needs no network (the +# host has one, but a job that fetches at run time is a job that fails at 2 am). Best-effort: +# without network at build time torchvision fetches lazily on the first run. +ENV TORCH_HOME=/app/torch-home +RUN uv run python -c "import torchvision.models as m; m.resnet18(weights=m.ResNet18_Weights.IMAGENET1K_V1); m.mobilenet_v3_small(weights=m.MobileNet_V3_Small_Weights.IMAGENET1K_V1)" \ + || echo "[build] backbone weights not pre-warmed (no network) — fetched on first run" + +RUN useradd --system --create-home --uid 999 trainer \ + && mkdir -p /data /out && chown -R trainer:trainer /app /out +USER trainer + +ENV TRAINER_DATA_DIR=/data \ + TRAINER_OUT_DIR=/out +VOLUME ["/out"] +ENTRYPOINT ["uv", "run", "--no-sync", "parking-trainer"] +CMD ["inspect"] diff --git a/apps/trainer/README.md b/apps/trainer/README.md new file mode 100644 index 0000000..4c2ad57 --- /dev/null +++ b/apps/trainer/README.md @@ -0,0 +1,36 @@ +# parking-trainer + +The phase-B **body-type classifier** job. Reads the wash collector's volume +(`collector.sqlite` + `crops/`), trains a classifier on the reviewer's labels, and writes a +versioned model folder the vision image bakes in — or refuses when validation is below the +floor. Design and decisions: `wiki/decisions/bodytype-classifier-training.md`. + +``` +parking-trainer inspect --data /data # what a run would train on +parking-trainer train --data /data --out /out # features mode (minutes) +parking-trainer train --mode finetune --epochs 12 ... # full fine-tune (about an hour on 4 cores) +parking-trainer evaluate --model /out//bodytype.onnx --data /data +parking-trainer publish /out/ --url https://git.infra.msai.al/api/packages/mca/generic/parking-bodytype +``` + +Exit codes: `0` model written · `2` not enough labels · `3` below the floor (report written, +no model) · `1` other. + +A passing run writes `//`: + +| file | what | +| --- | --- | +| `bodytype.onnx` | the classifier; input `image` = RGB float32 0–255 `[N,3,S,S]`, output `logits` `[N,K]`; normalisation is inside the graph | +| `bodytype.json` | sidecar: version, class list (in vocabulary order), input size, crop margin, backbone, mode, label counts, validation metrics | +| `report.md` | the human report: accuracy, per-class recall/precision, confusion matrix, dropped classes, loss weights | +| `metrics.json` | the same numbers, machine-readable | + +On the reviewer's host (the `wash-collector` stack): + +``` +docker compose -f docker-compose.collector.yml --profile train run --rm trainer inspect +docker compose -f docker-compose.collector.yml --profile train run --rm trainer train --min-accuracy 0.85 +``` + +Local dev: `uv sync --extra train` (CPU torch, ~200 MB), `uv run pytest -q`. The test suite +runs without the extra (torch tests skip), matching CI. diff --git a/apps/trainer/package.json b/apps/trainer/package.json new file mode 100644 index 0000000..52c7110 --- /dev/null +++ b/apps/trainer/package.json @@ -0,0 +1,13 @@ +{ + "name": "@parking/trainer", + "version": "0.0.0", + "private": true, + "//": "Thin shim so this Python job is a node in the Turbo task graph (NOT a JS package — deps are managed by uv/pyproject.toml). It is a one-off job image, never a booth service: see wiki/decisions/bodytype-classifier-training.md.", + "scripts": { + "lint": "uv run ruff check .", + "format": "uv run ruff format .", + "typecheck": "uv run mypy trainer", + "test": "uv run pytest -q", + "build": "echo 'no build step (Python job; see Dockerfile)'" + } +} diff --git a/apps/trainer/pyproject.toml b/apps/trainer/pyproject.toml new file mode 100644 index 0000000..aac2b55 --- /dev/null +++ b/apps/trainer/pyproject.toml @@ -0,0 +1,73 @@ +[project] +name = "parking-trainer" +version = "0.0.0" +description = "Phase-B body-type classifier trainer: reviewer labels + crops off the wash collector's volume → an ONNX classifier the vision image bakes in." +requires-python = ">=3.10,<4.0" +# Core deps are LIGHT on purpose (same rule as the vision service): `inspect`, `evaluate` +# and the data/report code run with only these, so `uv sync` and the test suite work +# in CI without the PyTorch stack. Training itself needs the `train` extra. +# See wiki/decisions/bodytype-classifier-training.md. +dependencies = [ + "numpy>=1.26", + # OpenCV does the decode + resize on BOTH sides (trainer and vision service): same + # library, same interpolation, same pixels — the preprocessing contract (preprocess.py). + "opencv-python-headless>=4.10", + "onnxruntime>=1.19", +] + +[project.scripts] +parking-trainer = "trainer.cli:main" + +[project.optional-dependencies] +# The training stack. CPU-only PyTorch (the reviewer's host has no usable GPU — the +# decision is recorded in the wiki page above): resolved from PyTorch's CPU wheel index, +# ~200 MB instead of the ~5 GB CUDA build. Install with: uv sync --extra train +# torch / torchvision are BSD-3; the ImageNet backbone weights ship under the same +# licence (the licence rule applies to weights as much as code). +train = [ + "torch>=2.4", + "torchvision>=0.19", + "onnx>=1.16", + "onnxscript>=0.3", # the torch.export-based ONNX exporter (MIT) +] + +[dependency-groups] +dev = [ + "ruff>=0.8", + "pytest>=8.3", + "mypy>=1.13", +] + +[tool.uv] +# Pick the CPU wheels for torch/torchvision from PyTorch's own index; everything else +# from PyPI. `explicit = true` keeps the index from shadowing PyPI for other packages. +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + +[tool.uv.sources] +torch = [{ index = "pytorch-cpu" }] +torchvision = [{ index = "pytorch-cpu" }] + +[tool.ruff] +line-length = 110 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.mypy] +python_version = "3.12" +strict = true +ignore_missing_imports = true + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["trainer"] diff --git a/apps/trainer/tests/conftest.py b/apps/trainer/tests/conftest.py new file mode 100644 index 0000000..a5b7e17 --- /dev/null +++ b/apps/trainer/tests/conftest.py @@ -0,0 +1,102 @@ +"""A synthetic collector volume: the collector's `items` table (same DDL as apps/collector +src/db.ts) + JPEG crops. Classes are told apart by COLOUR so even a random-init backbone's +features separate them — the tests check the plumbing (split, floor, export, sidecar), +not accuracy on real cars.""" + +from __future__ import annotations + +import sqlite3 +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import numpy as np +import pytest + +DDL = """ +CREATE TABLE IF NOT EXISTS items ( + id TEXT PRIMARY KEY, booth TEXT NOT NULL, kind TEXT NOT NULL DEFAULT 'wash', + order_ref TEXT NOT NULL, at TEXT NOT NULL, operator_ref TEXT NOT NULL DEFAULT '', + operator_category_id TEXT NOT NULL DEFAULT '', operator_category_name TEXT NOT NULL DEFAULT '', + operator_classes TEXT NOT NULL DEFAULT '[]', service TEXT NOT NULL, vision_class TEXT NOT NULL, + vision_confidence REAL NOT NULL, vision_category_id TEXT, downgraded INTEGER NOT NULL DEFAULT 0, + image_width INTEGER NOT NULL, image_height INTEGER NOT NULL, plate_blurred INTEGER NOT NULL, + image_path TEXT NOT NULL, received_at TEXT NOT NULL, review_label TEXT, reviewed_at TEXT, reviewer TEXT +); +""" + +COLOURS = {"sedan": (200, 40, 40), "suv": (40, 200, 40), "van": (40, 40, 200), "truck": (200, 200, 40)} + + +def write_jpeg(path: Path, colour: tuple[int, int, int], rng: np.random.Generator) -> None: + import cv2 + + path.parent.mkdir(parents=True, exist_ok=True) + h, w = int(rng.integers(120, 200)), int(rng.integers(160, 260)) + img = np.empty((h, w, 3), np.uint8) + img[:] = colour[::-1] # BGR + noise = rng.integers(-20, 20, size=img.shape, dtype=np.int16) + img = np.clip(img.astype(np.int16) + noise, 0, 255).astype(np.uint8) + cv2.imwrite(str(path), img, [cv2.IMWRITE_JPEG_QUALITY, 85]) + + +@pytest.fixture +def collector_dir(tmp_path: Path) -> Path: + """40 labelled crops per class for sedan/suv/van, 5 for truck (below the minimum), a few + unusable, a few pending, one labelled row whose file is missing.""" + rng = np.random.default_rng(1) + con = sqlite3.connect(tmp_path / "collector.sqlite") + con.executescript(DDL) + t0 = datetime(2026, 9, 1, tzinfo=timezone.utc) + n = 0 + + def add(label: str | None, reviewed: bool, kind: str = "wash", missing: bool = False) -> None: + nonlocal n + n += 1 + item = f"item-{n:04d}" + rel = f"crops/booth-2/{item}.jpg" + colour = COLOURS.get(label or "sedan", (128, 128, 128)) + if not missing: + write_jpeg(tmp_path / rel, colour, rng) + at = (t0 + timedelta(minutes=10 * n)).isoformat().replace("+00:00", "Z") + reviewed_at = ( + (t0 + timedelta(days=1, minutes=n)).isoformat().replace("+00:00", "Z") if reviewed else None + ) + con.execute( + "INSERT INTO items (id, booth, kind, order_ref, at, service, vision_class, vision_confidence, " + "image_width, image_height, plate_blurred, image_path, received_at, review_label, " + "reviewed_at, reviewer) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + item, + "booth-2", + kind, + "o", + at, + "wash", + "car" if label != "truck" else "truck", + 0.9, + 200, + 150, + 1, + rel, + at, + label if reviewed else None, + reviewed_at, + "reviewer" if reviewed else None, + ), + ) + + # Interleaved in time so every class exists on both sides of the time split. + for i in range(40): + for label in ("sedan", "suv", "van"): + add(label, True) + if i % 8 == 0: + add("truck", True) + add("unusable", True) + add("unusable", True) + add("sedan", True, missing=True) + for _ in range(6): + add(None, False, kind="entry") + con.commit() + con.close() + return tmp_path diff --git a/apps/trainer/tests/test_data.py b/apps/trainer/tests/test_data.py new file mode 100644 index 0000000..e069910 --- /dev/null +++ b/apps/trainer/tests/test_data.py @@ -0,0 +1,98 @@ +"""Data rules, torch-free: labels, the time split, thin classes, weights, the report.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from trainer.cli import main +from trainer.data import ( + class_weights, + load_labelled, + load_reviewed_since, + load_unlabelled, + make_split, + summarise, +) +from trainer.preprocess import CROP_MARGIN, Sidecar, load_input +from trainer.report import compute_metrics, render_report + + +def test_loads_only_reviewed_usable_rows_with_a_crop_on_disk(collector_dir: Path) -> None: + samples, missing = load_labelled(collector_dir) + assert missing == 1 # the labelled row whose file is gone + assert len(samples) == 125 # 3×40 + 5 trucks; unusable and pending excluded + assert all(s.path.is_file() for s in samples) + assert {s.label for s in samples} == {"sedan", "suv", "van", "truck"} + assert summarise(samples)["byClass"] == {"sedan": 40, "suv": 40, "van": 40, "truck": 5} + assert len(load_unlabelled(collector_dir)) == 6 + assert len(load_reviewed_since(collector_dir, "2026-09-02T00:00:00Z")) == 125 + assert load_reviewed_since(collector_dir, "2030-01-01T00:00:00Z") == [] + + +def test_split_is_by_time_and_drops_thin_classes(collector_dir: Path) -> None: + samples, _ = load_labelled(collector_dir) + split = make_split(samples, val_fraction=0.2, min_per_class=20) + assert split.classes == ("sedan", "suv", "van") # canonical order, truck dropped + assert split.dropped == {"truck": 5} + assert len(split.train) + len(split.val) == 120 + assert len(split.val) == 24 + assert max(s.at for s in split.train) < min(s.at for s in split.val) # newest = validation + assert all(v > 0 for v in split.counts("val").values()) + + +def test_class_weights_lean_against_imbalance_but_gently(collector_dir: Path) -> None: + samples, _ = load_labelled(collector_dir) + vans = [s for s in samples if s.label == "van"] + keep = set(vans[::10]) # 4 of 40 vans survive + split = make_split([s for s in samples if s.label != "van" or s in keep], 0.2, 3) + w = dict(zip(split.classes, class_weights(split), strict=True)) + assert w["van"] > w["sedan"] > 0 # the rare class weighs more + assert w["van"] / w["sedan"] < 4 # but not the full inverse ratio (damped) + assert abs(sum(w.values()) / len(w) - 1.0) < 1e-9 + + +def test_metrics_and_report() -> None: + classes = ("sedan", "suv") + m = compute_metrics(classes, [0, 0, 1, 1], [0, 1, 1, 1], camera=["car"] * 4) + assert m.accuracy == 0.75 + assert m.per_class["sedan"].recall == 0.5 and m.per_class["suv"].precision == 2 / 3 + assert m.confusion == [[1, 1], [0, 2]] + assert m.camera_agreement == 0.0 + text = render_report( + version="v1", + trained_at="t", + mode="features", + backbone="resnet18", + epochs=3, + classes=classes, + train_counts={"sedan": 10, "suv": 8}, + val_counts={"sedan": 2, "suv": 2}, + dropped={"truck": 2}, + missing_files=1, + weights=[0.9, 1.1], + metrics=m, + min_accuracy=0.85, + written=False, + ) + assert "MODEL NOT WRITTEN" in text and "| **sedan** | 1 | 1 |" in text and "truck (2)" in text + + +def test_preprocess_contract(tmp_path: Path, collector_dir: Path) -> None: + samples, _ = load_labelled(collector_dir) + x = load_input(samples[0].path, 32) + assert x.shape == (3, 32, 32) and x.dtype.name == "float32" and 0 <= x.min() and x.max() <= 255 + assert x[0].mean() > x[2].mean() # a sedan crop is red: RGB order, not BGR + assert load_input(tmp_path / "nope.jpg", 32) is None + side = Sidecar(version="v1", classes=["sedan", "suv"]) + side.write(tmp_path / "s.json") + back = Sidecar.read(tmp_path / "s.json") + assert back == side and back.crop_margin == CROP_MARGIN == 0.08 and back.normalization == "in-graph" + + +def test_inspect_prints_the_run_shape(collector_dir: Path, capsys) -> None: # type: ignore[no-untyped-def] + assert main(["inspect", "--data", str(collector_dir)]) == 0 + out = json.loads(capsys.readouterr().out) + assert out["ready"] is True and out["run"]["classes"] == ["sedan", "suv", "van"] + assert out["run"]["dropped"] == {"truck": 5} and out["missingCrops"] == 1 + assert main(["inspect", "--data", str(collector_dir), "--min-per-class", "100"]) == 2 diff --git a/apps/trainer/tests/test_train.py b/apps/trainer/tests/test_train.py new file mode 100644 index 0000000..866cde5 --- /dev/null +++ b/apps/trainer/tests/test_train.py @@ -0,0 +1,155 @@ +"""The training job end to end on the synthetic volume — needs the `train` extra (torch); +skipped where it is not installed (CI syncs without it, like the vision service).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") + +from trainer.cli import main # noqa: E402 +from trainer.infer import OnnxClassifier # noqa: E402 +from trainer.preprocess import Sidecar # noqa: E402 + +COMMON = ["--no-pretrained", "--input-size", "64", "--no-cache", "--seed", "3"] + + +def test_features_run_writes_model_sidecar_report_and_evaluates( + collector_dir: Path, tmp_path: Path, capsys +) -> None: # type: ignore[no-untyped-def] + out = tmp_path / "out" + rc = main( + [ + "train", + "--data", + str(collector_dir), + "--out", + str(out), + "--version", + "vtest", + "--mode", + "features", + "--epochs", + "150", + "--min-accuracy", + "0.0", + *COMMON, + ] + ) + assert rc == 0 + d = out / "vtest" + assert {p.name for p in d.iterdir()} == {"bodytype.onnx", "bodytype.json", "report.md", "metrics.json"} + side = Sidecar.read(d / "bodytype.json") + assert side.classes == ["sedan", "suv", "van"] and side.input_size == 64 and side.mode == "features" + assert side.labels == {"train": 96, "val": 24} and side.metrics["floor"] == 0.0 + metrics = json.loads((d / "metrics.json").read_text()) + assert metrics["n"] == 24 and metrics["onnx_agreement"] == 1.0 + # Colour-coded classes: even a random backbone's pooled features separate them. + assert metrics["accuracy"] >= 0.9 + report = (d / "report.md").read_text() + assert ( + "MODEL WRITTEN" in report + and "truck (5)" in report + and "crop is missing on disk (skipped): 1" in report + ) + + # The exported graph takes raw 0–255 RGB and answers by itself. + clf = OnnxClassifier(d / "bodytype.onnx") + probs, kept = clf.predict_files( + [s for s in sorted((collector_dir / "crops" / "booth-2").glob("*.jpg"))][:6] + ) + assert probs.shape == (6, 3) and kept == [0, 1, 2, 3, 4, 5] + assert np.allclose(probs.sum(axis=1), 1.0, atol=1e-4) + + # evaluate: labels reviewed after training (none — the fixture's reviews predate it) and the + # unlabelled pile (6 entry samples). + capsys.readouterr() + assert main(["evaluate", "--data", str(collector_dir), "--model", str(d / "bodytype.onnx")]) == 0 + res = json.loads(capsys.readouterr().out) + assert res["model"] == "vtest" and res["reviewedSince"] is None + assert res["unlabelled"]["n"] == 6 and sum(res["unlabelled"]["predicted"].values()) == 6 + assert ( + main( + [ + "evaluate", + "--data", + str(collector_dir), + "--model", + str(d / "bodytype.onnx"), + "--since", + "2026-09-01T00:00:00Z", + ] + ) + == 0 + ) + res2 = json.loads(capsys.readouterr().out) + assert res2["reviewedSince"]["n"] == 125 - 5 # trucks are not a class the model knows + + +def test_below_the_floor_writes_the_report_but_no_model(collector_dir: Path, tmp_path: Path) -> None: + out = tmp_path / "out" + rc = main( + [ + "train", + "--data", + str(collector_dir), + "--out", + str(out), + "--version", + "vlow", + "--mode", + "features", + "--epochs", + "5", + "--min-accuracy", + "1.01", + *COMMON, + ] + ) + assert rc == 3 + d = out / "vlow" + assert {p.name for p in d.iterdir()} == {"report.md", "metrics.json"} + assert "MODEL NOT WRITTEN" in (d / "report.md").read_text() + + +def test_not_enough_labels_is_exit_2(collector_dir: Path, tmp_path: Path) -> None: + out = tmp_path / "out" + rc = main(["train", "--data", str(collector_dir), "--out", str(out), "--min-per-class", "100", *COMMON]) + assert rc == 2 + assert not out.exists() + + +def test_finetune_runs_and_uses_the_feature_cache(collector_dir: Path, tmp_path: Path) -> None: + out = tmp_path / "out" + args = [ + "train", + "--data", + str(collector_dir), + "--out", + str(out), + "--mode", + "finetune", + "--backbone", + "mobilenet_v3_small", + "--epochs", + "1", + "--batch", + "16", + "--min-accuracy", + "0.0", + "--no-pretrained", + "--input-size", + "64", + "--seed", + "3", + ] + assert main([*args, "--version", "vft"]) == 0 + cache = out / "cache" / "features-mobilenet_v3_small-64.npz" + assert cache.exists() + z = np.load(cache) + assert len(z["ids"]) == 96 and z["feats"].shape == (96, 576) + assert Sidecar.read(out / "vft" / "bodytype.json").mode == "finetune" diff --git a/apps/trainer/trainer/__init__.py b/apps/trainer/trainer/__init__.py new file mode 100644 index 0000000..e65cc4f --- /dev/null +++ b/apps/trainer/trainer/__init__.py @@ -0,0 +1,7 @@ +"""parking-trainer — the phase-B body-type classifier job. + +Reads the wash collector's SQLite + crops straight off its volume, splits by TIME, trains a +small classifier on a pretrained backbone, and writes the ONNX model + sidecar + report — +or refuses to write the model when validation is below the owner's floor. +See wiki/decisions/bodytype-classifier-training.md. +""" diff --git a/apps/trainer/trainer/cli.py b/apps/trainer/trainer/cli.py new file mode 100644 index 0000000..f5af89a --- /dev/null +++ b/apps/trainer/trainer/cli.py @@ -0,0 +1,388 @@ +"""parking-trainer — inspect / train / evaluate / publish. + + parking-trainer inspect --data /data + parking-trainer train --data /data --out /out [--mode features|finetune] [--min-accuracy 0.85] + parking-trainer evaluate --model /out//bodytype.onnx --data /data + parking-trainer publish /out/ --url https:///api/packages//generic/parking-bodytype + +Exit codes: 0 ok · 2 not enough labels · 3 trained but below the floor (report written, model +NOT written) · 1 anything else. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +from .data import ( + VEHICLE_CLASSES, + class_weights, + load_labelled, + load_reviewed_since, + load_unlabelled, + make_split, + suggested_epochs, + summarise, +) +from .preprocess import CROP_MARGIN, Sidecar +from .report import compute_metrics, render_report + +MODEL_FILE = "bodytype.onnx" +SIDECAR_FILE = "bodytype.json" +REPORT_FILE = "report.md" +METRICS_FILE = "metrics.json" + + +def _log(msg: str) -> None: + print(f"[trainer] {msg}", file=sys.stderr, flush=True) + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +# ---------------------------------------------------------------------------------- +# inspect +# ---------------------------------------------------------------------------------- + + +def cmd_inspect(a: argparse.Namespace) -> int: + samples, missing = load_labelled(a.data) + split = make_split(samples, a.val_fraction, a.min_per_class) + out = { + "labelled": summarise(samples), + "missingCrops": missing, + "run": { + "classes": list(split.classes), + "train": split.counts("train"), + "val": split.counts("val"), + "dropped": split.dropped, + "minPerClass": a.min_per_class, + "valFraction": a.val_fraction, + }, + "ready": len(split.classes) >= 2, + } + print(json.dumps(out, indent=2)) + return 0 if out["ready"] else 2 + + +# ---------------------------------------------------------------------------------- +# train +# ---------------------------------------------------------------------------------- + + +def cmd_train(a: argparse.Namespace) -> int: + try: + from . import model as M + except ImportError as exc: # torch missing + _log(f"the training stack is not installed ({exc}); install with: uv sync --extra train") + return 1 + import numpy as np + + samples, missing = load_labelled(a.data) + split = make_split(samples, a.val_fraction, a.min_per_class) + if len(split.classes) < 2: + _log( + f"not enough labels: {len(samples)} usable, classes with >= {a.min_per_class}: " + f"{list(split.classes)} (dropped {split.dropped}); nothing to train" + ) + return 2 + version = a.version or datetime.now(timezone.utc).strftime("v%Y%m%d-%H%M") + out_dir = a.out / version + epochs = a.epochs or suggested_epochs(len(split.train), a.mode) + weights = class_weights(split) + idx = split.class_index + _log( + f"{version}: {a.mode} on {a.backbone}, classes {list(split.classes)}, " + f"{len(split.train)} train / {len(split.val)} val, {epochs} epochs" + ) + + t0 = time.perf_counter() + x_train, train = M.load_images(split.train, a.input_size) + x_val, val = M.load_images(split.val, a.input_size) + y_train = [idx[s.label] for s in train] + y_val = [idx[s.label] for s in val] + _log(f"decoded {len(train)} + {len(val)} crops in {time.perf_counter() - t0:.1f}s") + if len(val) == 0 or len(set(y_train)) < 2: + _log("not enough decodable crops on both sides of the split") + return 2 + + backbone = M.build_backbone(a.backbone, pretrained=not a.no_pretrained) + cache = ( + None if a.no_cache else M.FeatureCache(a.out / "cache" / f"features-{a.backbone}-{a.input_size}.npz") + ) + t0 = time.perf_counter() + f_train = M.features_for(backbone, train, x_train, cache) + _log( + f"features for {len(train)} train crops in {time.perf_counter() - t0:.1f}s " + f"(cache: {cache.path if cache else 'off'})" + ) + head_epochs = epochs if a.mode == "features" else max(30, epochs * 5) + head = M.train_head(f_train, y_train, len(split.classes), weights, head_epochs, seed=a.seed) + net = M.Classifier.make(backbone, head) + + if a.mode == "finetune": + t0 = time.perf_counter() + net = M.finetune( + net, x_train, y_train, weights, epochs, batch=a.batch, lr=a.lr, seed=a.seed, log=_log + ) + _log(f"fine-tuned in {(time.perf_counter() - t0) / 60:.1f} min") + + logits = M.predict_logits(net, x_val) + y_pred = logits.argmax(axis=1).tolist() + metrics = compute_metrics(split.classes, y_val, y_pred, camera=[s.vision_class for s in val]) + + # Export and check the graph gives the same answers as the torch model. + out_dir.mkdir(parents=True, exist_ok=True) + tmp_model = out_dir / (MODEL_FILE + ".tmp") + M.export_onnx(net, a.input_size, tmp_model) + from .infer import OnnxClassifier + + sidecar = Sidecar( + version=version, + classes=list(split.classes), + input_size=a.input_size, + backbone=a.backbone, + mode=a.mode, + trained_at=_now(), + labels={"train": len(train), "val": len(val)}, + ) + tmp_side = out_dir / (SIDECAR_FILE + ".tmp") + sidecar.write(tmp_side) + onnx_pred = OnnxClassifier(tmp_model, tmp_side).predict_inputs(x_val.astype(np.float32)).argmax(axis=1) + metrics.onnx_agreement = float((onnx_pred == np.array(y_pred)).mean()) if len(y_pred) else None + sidecar.metrics = { + "accuracy": metrics.accuracy, + "macroRecall": metrics.macro_recall, + "perClass": {c: m.__dict__ for c, m in metrics.per_class.items()}, + "floor": a.min_accuracy, + } + + written = metrics.accuracy >= a.min_accuracy and (metrics.onnx_agreement or 0.0) >= 0.99 + notes = [] + if metrics.onnx_agreement is not None and metrics.onnx_agreement < 0.99: + notes.append( + f"ONNX export disagrees with the torch model ({metrics.onnx_agreement:.3f}); model withheld" + ) + report = render_report( + version=version, + trained_at=sidecar.trained_at, + mode=a.mode, + backbone=a.backbone, + epochs=epochs, + classes=split.classes, + train_counts=split.counts("train"), + val_counts=split.counts("val"), + dropped=split.dropped, + missing_files=missing, + weights=weights, + metrics=metrics, + min_accuracy=a.min_accuracy, + written=written, + notes=notes, + ) + (out_dir / REPORT_FILE).write_text(report) + (out_dir / METRICS_FILE).write_text(json.dumps(metrics.to_dict(), indent=2) + "\n") + if written: + sidecar.write(out_dir / SIDECAR_FILE) + tmp_model.replace(out_dir / MODEL_FILE) + tmp_side.unlink(missing_ok=True) + _log( + f"MODEL WRITTEN: {out_dir / MODEL_FILE} " + f"(accuracy {metrics.accuracy:.3f} >= floor {a.min_accuracy})" + ) + else: + tmp_model.unlink(missing_ok=True) + tmp_side.unlink(missing_ok=True) + _log( + f"MODEL NOT WRITTEN: accuracy {metrics.accuracy:.3f} < floor {a.min_accuracy}; " + f"see {out_dir / REPORT_FILE}" + ) + print(report) + return 0 if written else 3 + + +# ---------------------------------------------------------------------------------- +# evaluate — an existing model against labels that arrived AFTER it was trained, and its +# view of the unlabelled pile (the ongoing accuracy check without labelling everything) +# ---------------------------------------------------------------------------------- + + +def cmd_evaluate(a: argparse.Namespace) -> int: + from .infer import OnnxClassifier + + clf = OnnxClassifier(a.model) + since = a.since or clf.sidecar.trained_at + classes = clf.classes + result: dict[str, object] = {"model": clf.sidecar.version, "classes": classes, "since": since} + + reviewed = [s for s in load_reviewed_since(a.data, since) if s.label in classes] + if reviewed: + probs, kept = clf.predict_files([s.path for s in reviewed]) + rows = [reviewed[i] for i in kept] + y_true = [classes.index(s.label) for s in rows] + y_pred = probs.argmax(axis=1).tolist() + m = compute_metrics(classes, y_true, y_pred, camera=[s.vision_class for s in rows]) + result["reviewedSince"] = m.to_dict() + else: + result["reviewedSince"] = None + + pending = load_unlabelled(a.data, a.limit) + if pending: + probs, kept = clf.predict_files([s.path for s in pending]) + rows = [pending[i] for i in kept] + pred = probs.argmax(axis=1) + conf = probs.max(axis=1) + hist = {c: int((pred == i).sum()) for i, c in enumerate(classes)} + result["unlabelled"] = { + "n": len(rows), + "predicted": hist, + "meanConfidence": float(conf.mean()) if len(rows) else None, + "belowHalf": int((conf < 0.5).sum()), + "agreesWithDetector": float( + sum(1 for p, s in zip(pred, rows, strict=True) if classes[int(p)] == s.vision_class) + / len(rows) + ) + if rows + else None, + } + else: + result["unlabelled"] = None + print(json.dumps(result, indent=2)) + return 0 + + +# ---------------------------------------------------------------------------------- +# publish — the versioned files to a Gitea generic package (weights are not code, they +# do not live in git; the vision image fetches them by URL at build) +# ---------------------------------------------------------------------------------- + + +def cmd_publish(a: argparse.Namespace) -> int: + d: Path = a.dir + files = [d / MODEL_FILE, d / SIDECAR_FILE, d / REPORT_FILE, d / METRICS_FILE] + for f in files[:2]: + if not f.exists(): + _log(f"{f} missing — nothing to publish (a run below the floor writes no model)") + return 1 + version = Sidecar.read(d / SIDECAR_FILE).version + token = a.token or os.environ.get("TRAINER_PUBLISH_TOKEN", "") + if not token: + _log("no token: pass --token or set TRAINER_PUBLISH_TOKEN") + return 1 + base = a.url.rstrip("/") + "/" + version + for f in files: + if not f.exists(): + continue + req = urllib.request.Request(f"{base}/{f.name}", data=f.read_bytes(), method="PUT") + req.add_header("Authorization", f"token {token}") + req.add_header("Content-Type", "application/octet-stream") + with urllib.request.urlopen(req, timeout=120) as r: + _log(f"PUT {base}/{f.name} → {r.status}") + _log(f"published {version}; pin it in apps/vision/models/bodytype.version and rebuild the vision image") + return 0 + + +# ---------------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="parking-trainer", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + sub = p.add_subparsers(dest="cmd", required=True) + + def data_args(sp: argparse.ArgumentParser) -> None: + sp.add_argument( + "--data", + type=Path, + default=Path(os.environ.get("TRAINER_DATA_DIR", "/data")), + help="collector volume (collector.sqlite + crops/)", + ) + + def split_args(sp: argparse.ArgumentParser) -> None: + sp.add_argument( + "--min-per-class", + type=int, + default=20, + help="classes with fewer reviewed crops are dropped from the run", + ) + sp.add_argument( + "--val-fraction", type=float, default=0.2, help="newest fraction held out for validation" + ) + + i = sub.add_parser("inspect", help="what a run would train on") + data_args(i) + split_args(i) + i.set_defaults(fn=cmd_inspect) + + t = sub.add_parser("train", help="train, evaluate, export (or refuse)") + data_args(t) + split_args(t) + t.add_argument( + "--out", + type=Path, + default=Path(os.environ.get("TRAINER_OUT_DIR", "/out")), + help="output root; a / folder is created under it", + ) + t.add_argument("--mode", choices=["features", "finetune"], default="features") + t.add_argument( + "--backbone", choices=["resnet18", "mobilenet_v3_small", "efficientnet_b0"], default="resnet18" + ) + t.add_argument("--epochs", type=int, default=0, help="0 = pick from the data size") + t.add_argument("--batch", type=int, default=32) + t.add_argument("--lr", type=float, default=1e-4, help="fine-tune learning rate") + t.add_argument("--input-size", type=int, default=224) + t.add_argument( + "--min-accuracy", type=float, default=0.85, help="validation floor below which NO model is written" + ) + t.add_argument("--version", default="", help="model version (default v-