feat(trainer): phase-B body-type classifier — trainer job on the collector host + the classifier stage on the booth
Build & push images / images (push) Successful in 6m31s
Build & push images / images (push) Successful in 6m31s
apps/trainer (parking-trainer): inspect / train / evaluate / publish. Reads the wash collector's SQLite + crops read-only off its volume; time split (validation = newest slice); thin classes dropped; damped class weights; `features` mode (frozen ImageNet backbone, on-disk feature cache, seconds to retrain) and `finetune` mode (light augmentation). CPU-only torch from PyTorch's wheel index. ONNX export checked against the torch model; NO model file below the validation floor (exit 3, report still written); exit 2 = not enough labels. `evaluate` scores a shipped model on labels reviewed after training + the unlabelled pile; `publish` PUTs a version folder to a Gitea generic package. Light core deps; the `train` extra is heavy — CI syncs without it, torch tests skip. apps/vision: BodyTypeClassifier (bodytype.onnx + sidecar = the preprocessing contract: crop margin, input size, RGB 0-255, normalisation inside the graph) and RefinedVehicleDetector over YOLOX — refines only `car` or a class the classifier trained on, min-confidence, `detector_class` on the result; path set but no file = phase B off without an error; a broken file is a health detail. models/bodytype.version (tracked, empty) pins the published version the Dockerfile fetches at build (BuildKit secret; a pin that cannot be fetched fails the build). Verified: a trainer model gives identical probabilities inside the vision service; both images built and smoke-tested. Delivery: parking-trainer image in build-images.yml, the `trainer` compose profile on the collector stack (CPU, read-only data, TRAINER_OUT), commented TRAINER_OUT/PUBLISH_TOKEN in the wash-collector stack, .dockerignore for both Python contexts, trainer deps synced in CI. Wiki: bodytype-classifier-training rewritten as built (+ one fleet model not per site, secrets/access, where the crops live), opencv-anpr-service §Phase B, vision-review-outbox, vision-service-packaging, fleet-deployment-komodo, index, log. Claude-Session: https://claude.ai/code/session_01FWncR69HgGPuei1dLrW3cU
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
"""A synthetic collector volume: the collector's `items` table (same DDL as apps/collector
|
||||
src/db.ts) + JPEG crops. Classes are told apart by COLOUR so even a random-init backbone's
|
||||
features separate them — the tests check the plumbing (split, floor, export, sidecar),
|
||||
not accuracy on real cars."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
DDL = """
|
||||
CREATE TABLE IF NOT EXISTS items (
|
||||
id TEXT PRIMARY KEY, booth TEXT NOT NULL, kind TEXT NOT NULL DEFAULT 'wash',
|
||||
order_ref TEXT NOT NULL, at TEXT NOT NULL, operator_ref TEXT NOT NULL DEFAULT '',
|
||||
operator_category_id TEXT NOT NULL DEFAULT '', operator_category_name TEXT NOT NULL DEFAULT '',
|
||||
operator_classes TEXT NOT NULL DEFAULT '[]', service TEXT NOT NULL, vision_class TEXT NOT NULL,
|
||||
vision_confidence REAL NOT NULL, vision_category_id TEXT, downgraded INTEGER NOT NULL DEFAULT 0,
|
||||
image_width INTEGER NOT NULL, image_height INTEGER NOT NULL, plate_blurred INTEGER NOT NULL,
|
||||
image_path TEXT NOT NULL, received_at TEXT NOT NULL, review_label TEXT, reviewed_at TEXT, reviewer TEXT
|
||||
);
|
||||
"""
|
||||
|
||||
COLOURS = {"sedan": (200, 40, 40), "suv": (40, 200, 40), "van": (40, 40, 200), "truck": (200, 200, 40)}
|
||||
|
||||
|
||||
def write_jpeg(path: Path, colour: tuple[int, int, int], rng: np.random.Generator) -> None:
|
||||
import cv2
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
h, w = int(rng.integers(120, 200)), int(rng.integers(160, 260))
|
||||
img = np.empty((h, w, 3), np.uint8)
|
||||
img[:] = colour[::-1] # BGR
|
||||
noise = rng.integers(-20, 20, size=img.shape, dtype=np.int16)
|
||||
img = np.clip(img.astype(np.int16) + noise, 0, 255).astype(np.uint8)
|
||||
cv2.imwrite(str(path), img, [cv2.IMWRITE_JPEG_QUALITY, 85])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collector_dir(tmp_path: Path) -> Path:
|
||||
"""40 labelled crops per class for sedan/suv/van, 5 for truck (below the minimum), a few
|
||||
unusable, a few pending, one labelled row whose file is missing."""
|
||||
rng = np.random.default_rng(1)
|
||||
con = sqlite3.connect(tmp_path / "collector.sqlite")
|
||||
con.executescript(DDL)
|
||||
t0 = datetime(2026, 9, 1, tzinfo=timezone.utc)
|
||||
n = 0
|
||||
|
||||
def add(label: str | None, reviewed: bool, kind: str = "wash", missing: bool = False) -> None:
|
||||
nonlocal n
|
||||
n += 1
|
||||
item = f"item-{n:04d}"
|
||||
rel = f"crops/booth-2/{item}.jpg"
|
||||
colour = COLOURS.get(label or "sedan", (128, 128, 128))
|
||||
if not missing:
|
||||
write_jpeg(tmp_path / rel, colour, rng)
|
||||
at = (t0 + timedelta(minutes=10 * n)).isoformat().replace("+00:00", "Z")
|
||||
reviewed_at = (
|
||||
(t0 + timedelta(days=1, minutes=n)).isoformat().replace("+00:00", "Z") if reviewed else None
|
||||
)
|
||||
con.execute(
|
||||
"INSERT INTO items (id, booth, kind, order_ref, at, service, vision_class, vision_confidence, "
|
||||
"image_width, image_height, plate_blurred, image_path, received_at, review_label, "
|
||||
"reviewed_at, reviewer) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
item,
|
||||
"booth-2",
|
||||
kind,
|
||||
"o",
|
||||
at,
|
||||
"wash",
|
||||
"car" if label != "truck" else "truck",
|
||||
0.9,
|
||||
200,
|
||||
150,
|
||||
1,
|
||||
rel,
|
||||
at,
|
||||
label if reviewed else None,
|
||||
reviewed_at,
|
||||
"reviewer" if reviewed else None,
|
||||
),
|
||||
)
|
||||
|
||||
# Interleaved in time so every class exists on both sides of the time split.
|
||||
for i in range(40):
|
||||
for label in ("sedan", "suv", "van"):
|
||||
add(label, True)
|
||||
if i % 8 == 0:
|
||||
add("truck", True)
|
||||
add("unusable", True)
|
||||
add("unusable", True)
|
||||
add("sedan", True, missing=True)
|
||||
for _ in range(6):
|
||||
add(None, False, kind="entry")
|
||||
con.commit()
|
||||
con.close()
|
||||
return tmp_path
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Data rules, torch-free: labels, the time split, thin classes, weights, the report."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from trainer.cli import main
|
||||
from trainer.data import (
|
||||
class_weights,
|
||||
load_labelled,
|
||||
load_reviewed_since,
|
||||
load_unlabelled,
|
||||
make_split,
|
||||
summarise,
|
||||
)
|
||||
from trainer.preprocess import CROP_MARGIN, Sidecar, load_input
|
||||
from trainer.report import compute_metrics, render_report
|
||||
|
||||
|
||||
def test_loads_only_reviewed_usable_rows_with_a_crop_on_disk(collector_dir: Path) -> None:
|
||||
samples, missing = load_labelled(collector_dir)
|
||||
assert missing == 1 # the labelled row whose file is gone
|
||||
assert len(samples) == 125 # 3×40 + 5 trucks; unusable and pending excluded
|
||||
assert all(s.path.is_file() for s in samples)
|
||||
assert {s.label for s in samples} == {"sedan", "suv", "van", "truck"}
|
||||
assert summarise(samples)["byClass"] == {"sedan": 40, "suv": 40, "van": 40, "truck": 5}
|
||||
assert len(load_unlabelled(collector_dir)) == 6
|
||||
assert len(load_reviewed_since(collector_dir, "2026-09-02T00:00:00Z")) == 125
|
||||
assert load_reviewed_since(collector_dir, "2030-01-01T00:00:00Z") == []
|
||||
|
||||
|
||||
def test_split_is_by_time_and_drops_thin_classes(collector_dir: Path) -> None:
|
||||
samples, _ = load_labelled(collector_dir)
|
||||
split = make_split(samples, val_fraction=0.2, min_per_class=20)
|
||||
assert split.classes == ("sedan", "suv", "van") # canonical order, truck dropped
|
||||
assert split.dropped == {"truck": 5}
|
||||
assert len(split.train) + len(split.val) == 120
|
||||
assert len(split.val) == 24
|
||||
assert max(s.at for s in split.train) < min(s.at for s in split.val) # newest = validation
|
||||
assert all(v > 0 for v in split.counts("val").values())
|
||||
|
||||
|
||||
def test_class_weights_lean_against_imbalance_but_gently(collector_dir: Path) -> None:
|
||||
samples, _ = load_labelled(collector_dir)
|
||||
vans = [s for s in samples if s.label == "van"]
|
||||
keep = set(vans[::10]) # 4 of 40 vans survive
|
||||
split = make_split([s for s in samples if s.label != "van" or s in keep], 0.2, 3)
|
||||
w = dict(zip(split.classes, class_weights(split), strict=True))
|
||||
assert w["van"] > w["sedan"] > 0 # the rare class weighs more
|
||||
assert w["van"] / w["sedan"] < 4 # but not the full inverse ratio (damped)
|
||||
assert abs(sum(w.values()) / len(w) - 1.0) < 1e-9
|
||||
|
||||
|
||||
def test_metrics_and_report() -> None:
|
||||
classes = ("sedan", "suv")
|
||||
m = compute_metrics(classes, [0, 0, 1, 1], [0, 1, 1, 1], camera=["car"] * 4)
|
||||
assert m.accuracy == 0.75
|
||||
assert m.per_class["sedan"].recall == 0.5 and m.per_class["suv"].precision == 2 / 3
|
||||
assert m.confusion == [[1, 1], [0, 2]]
|
||||
assert m.camera_agreement == 0.0
|
||||
text = render_report(
|
||||
version="v1",
|
||||
trained_at="t",
|
||||
mode="features",
|
||||
backbone="resnet18",
|
||||
epochs=3,
|
||||
classes=classes,
|
||||
train_counts={"sedan": 10, "suv": 8},
|
||||
val_counts={"sedan": 2, "suv": 2},
|
||||
dropped={"truck": 2},
|
||||
missing_files=1,
|
||||
weights=[0.9, 1.1],
|
||||
metrics=m,
|
||||
min_accuracy=0.85,
|
||||
written=False,
|
||||
)
|
||||
assert "MODEL NOT WRITTEN" in text and "| **sedan** | 1 | 1 |" in text and "truck (2)" in text
|
||||
|
||||
|
||||
def test_preprocess_contract(tmp_path: Path, collector_dir: Path) -> None:
|
||||
samples, _ = load_labelled(collector_dir)
|
||||
x = load_input(samples[0].path, 32)
|
||||
assert x.shape == (3, 32, 32) and x.dtype.name == "float32" and 0 <= x.min() and x.max() <= 255
|
||||
assert x[0].mean() > x[2].mean() # a sedan crop is red: RGB order, not BGR
|
||||
assert load_input(tmp_path / "nope.jpg", 32) is None
|
||||
side = Sidecar(version="v1", classes=["sedan", "suv"])
|
||||
side.write(tmp_path / "s.json")
|
||||
back = Sidecar.read(tmp_path / "s.json")
|
||||
assert back == side and back.crop_margin == CROP_MARGIN == 0.08 and back.normalization == "in-graph"
|
||||
|
||||
|
||||
def test_inspect_prints_the_run_shape(collector_dir: Path, capsys) -> None: # type: ignore[no-untyped-def]
|
||||
assert main(["inspect", "--data", str(collector_dir)]) == 0
|
||||
out = json.loads(capsys.readouterr().out)
|
||||
assert out["ready"] is True and out["run"]["classes"] == ["sedan", "suv", "van"]
|
||||
assert out["run"]["dropped"] == {"truck": 5} and out["missingCrops"] == 1
|
||||
assert main(["inspect", "--data", str(collector_dir), "--min-per-class", "100"]) == 2
|
||||
@@ -0,0 +1,155 @@
|
||||
"""The training job end to end on the synthetic volume — needs the `train` extra (torch);
|
||||
skipped where it is not installed (CI syncs without it, like the vision service)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
from trainer.cli import main # noqa: E402
|
||||
from trainer.infer import OnnxClassifier # noqa: E402
|
||||
from trainer.preprocess import Sidecar # noqa: E402
|
||||
|
||||
COMMON = ["--no-pretrained", "--input-size", "64", "--no-cache", "--seed", "3"]
|
||||
|
||||
|
||||
def test_features_run_writes_model_sidecar_report_and_evaluates(
|
||||
collector_dir: Path, tmp_path: Path, capsys
|
||||
) -> None: # type: ignore[no-untyped-def]
|
||||
out = tmp_path / "out"
|
||||
rc = main(
|
||||
[
|
||||
"train",
|
||||
"--data",
|
||||
str(collector_dir),
|
||||
"--out",
|
||||
str(out),
|
||||
"--version",
|
||||
"vtest",
|
||||
"--mode",
|
||||
"features",
|
||||
"--epochs",
|
||||
"150",
|
||||
"--min-accuracy",
|
||||
"0.0",
|
||||
*COMMON,
|
||||
]
|
||||
)
|
||||
assert rc == 0
|
||||
d = out / "vtest"
|
||||
assert {p.name for p in d.iterdir()} == {"bodytype.onnx", "bodytype.json", "report.md", "metrics.json"}
|
||||
side = Sidecar.read(d / "bodytype.json")
|
||||
assert side.classes == ["sedan", "suv", "van"] and side.input_size == 64 and side.mode == "features"
|
||||
assert side.labels == {"train": 96, "val": 24} and side.metrics["floor"] == 0.0
|
||||
metrics = json.loads((d / "metrics.json").read_text())
|
||||
assert metrics["n"] == 24 and metrics["onnx_agreement"] == 1.0
|
||||
# Colour-coded classes: even a random backbone's pooled features separate them.
|
||||
assert metrics["accuracy"] >= 0.9
|
||||
report = (d / "report.md").read_text()
|
||||
assert (
|
||||
"MODEL WRITTEN" in report
|
||||
and "truck (5)" in report
|
||||
and "crop is missing on disk (skipped): 1" in report
|
||||
)
|
||||
|
||||
# The exported graph takes raw 0–255 RGB and answers by itself.
|
||||
clf = OnnxClassifier(d / "bodytype.onnx")
|
||||
probs, kept = clf.predict_files(
|
||||
[s for s in sorted((collector_dir / "crops" / "booth-2").glob("*.jpg"))][:6]
|
||||
)
|
||||
assert probs.shape == (6, 3) and kept == [0, 1, 2, 3, 4, 5]
|
||||
assert np.allclose(probs.sum(axis=1), 1.0, atol=1e-4)
|
||||
|
||||
# evaluate: labels reviewed after training (none — the fixture's reviews predate it) and the
|
||||
# unlabelled pile (6 entry samples).
|
||||
capsys.readouterr()
|
||||
assert main(["evaluate", "--data", str(collector_dir), "--model", str(d / "bodytype.onnx")]) == 0
|
||||
res = json.loads(capsys.readouterr().out)
|
||||
assert res["model"] == "vtest" and res["reviewedSince"] is None
|
||||
assert res["unlabelled"]["n"] == 6 and sum(res["unlabelled"]["predicted"].values()) == 6
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"evaluate",
|
||||
"--data",
|
||||
str(collector_dir),
|
||||
"--model",
|
||||
str(d / "bodytype.onnx"),
|
||||
"--since",
|
||||
"2026-09-01T00:00:00Z",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
res2 = json.loads(capsys.readouterr().out)
|
||||
assert res2["reviewedSince"]["n"] == 125 - 5 # trucks are not a class the model knows
|
||||
|
||||
|
||||
def test_below_the_floor_writes_the_report_but_no_model(collector_dir: Path, tmp_path: Path) -> None:
|
||||
out = tmp_path / "out"
|
||||
rc = main(
|
||||
[
|
||||
"train",
|
||||
"--data",
|
||||
str(collector_dir),
|
||||
"--out",
|
||||
str(out),
|
||||
"--version",
|
||||
"vlow",
|
||||
"--mode",
|
||||
"features",
|
||||
"--epochs",
|
||||
"5",
|
||||
"--min-accuracy",
|
||||
"1.01",
|
||||
*COMMON,
|
||||
]
|
||||
)
|
||||
assert rc == 3
|
||||
d = out / "vlow"
|
||||
assert {p.name for p in d.iterdir()} == {"report.md", "metrics.json"}
|
||||
assert "MODEL NOT WRITTEN" in (d / "report.md").read_text()
|
||||
|
||||
|
||||
def test_not_enough_labels_is_exit_2(collector_dir: Path, tmp_path: Path) -> None:
|
||||
out = tmp_path / "out"
|
||||
rc = main(["train", "--data", str(collector_dir), "--out", str(out), "--min-per-class", "100", *COMMON])
|
||||
assert rc == 2
|
||||
assert not out.exists()
|
||||
|
||||
|
||||
def test_finetune_runs_and_uses_the_feature_cache(collector_dir: Path, tmp_path: Path) -> None:
|
||||
out = tmp_path / "out"
|
||||
args = [
|
||||
"train",
|
||||
"--data",
|
||||
str(collector_dir),
|
||||
"--out",
|
||||
str(out),
|
||||
"--mode",
|
||||
"finetune",
|
||||
"--backbone",
|
||||
"mobilenet_v3_small",
|
||||
"--epochs",
|
||||
"1",
|
||||
"--batch",
|
||||
"16",
|
||||
"--min-accuracy",
|
||||
"0.0",
|
||||
"--no-pretrained",
|
||||
"--input-size",
|
||||
"64",
|
||||
"--seed",
|
||||
"3",
|
||||
]
|
||||
assert main([*args, "--version", "vft"]) == 0
|
||||
cache = out / "cache" / "features-mobilenet_v3_small-64.npz"
|
||||
assert cache.exists()
|
||||
z = np.load(cache)
|
||||
assert len(z["ids"]) == 96 and z["feats"].shape == (96, 576)
|
||||
assert Sidecar.read(out / "vft" / "bodytype.json").mode == "finetune"
|
||||
Reference in New Issue
Block a user