fix(collector,trainer): migrate an existing collector DB on open; trainer handlers answer 500 JSON
The reviewer host's collector.sqlite was created by an earlier build, before the `kind` column. CREATE TABLE IF NOT EXISTS shapes only a new database, so every query naming the column failed: the collector's /health (container unhealthy), every booth ingest, and the trainer's readiness — whose stdlib server printed the traceback and dropped the socket, which the collector could only render as "trainer not reachable: fetch failed". Nine days like that. - CollectorDb.#migrate(): PRAGMA table_info against the list of columns added since the first deploy; ALTER TABLE ADD COLUMN for each missing one (all nullable or defaulted). Append to that list whenever a column joins the CREATE. Test replays the original schema: health, ingest, stats, a legacy row reads back with the defaults. - Trainer Handler._guarded(): any unexpected exception → 500 JSON naming it, never a dropped connection; /health keeps answering. Test drives readiness against an old-schema DB. - The collector's training status proxy includes the trainer's error text. Wiki: the incident and the schema rule (vision-review-outbox), what the message means (bodytype-classifier-training), log. Deploy: the new collector migrates on start; nothing manual. Claude-Session: https://claude.ai/code/session_01FWncR69HgGPuei1dLrW3cU
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
@@ -109,3 +110,38 @@ def test_train_job_then_versions_and_report(api) -> None: # type: ignore[no-unt
|
||||
code, job2 = call("POST", "/jobs", {"kind": "evaluate", "version": "vapi"})
|
||||
wait_idle(Handler.jobs)
|
||||
assert call("GET", f"/jobs/{job2['id']}")[1]["status"] == "done"
|
||||
|
||||
|
||||
def test_a_handler_error_is_a_500_json_reply_not_a_dropped_connection(tmp_path: Path) -> None:
|
||||
"""A collector DB from before the `kind` column (art-docker-station, 2026-09-16): readiness
|
||||
raised sqlite3.OperationalError, the stdlib server printed the traceback and closed the
|
||||
socket, and the collector could only say "trainer not reachable: fetch failed". The
|
||||
handler must answer 500 JSON naming the error instead."""
|
||||
old = tmp_path / "old"
|
||||
old.mkdir()
|
||||
con = sqlite3.connect(old / "collector.sqlite")
|
||||
con.executescript(
|
||||
"CREATE TABLE items (id TEXT PRIMARY KEY, booth TEXT NOT NULL, order_ref TEXT NOT NULL,"
|
||||
" at TEXT NOT NULL, service TEXT NOT NULL, vision_class TEXT NOT NULL,"
|
||||
" vision_confidence REAL NOT NULL, image_width INTEGER NOT NULL, image_height INTEGER NOT NULL,"
|
||||
" plate_blurred INTEGER NOT NULL, image_path TEXT NOT NULL, received_at TEXT NOT NULL,"
|
||||
" review_label TEXT, reviewed_at TEXT, reviewer TEXT)"
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
Handler.jobs = Jobs(old, tmp_path / "out", "https://example.invalid/pkg", "tok")
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
try:
|
||||
req = urllib.request.Request(f"http://127.0.0.1:{httpd.server_address[1]}/readiness")
|
||||
with pytest.raises(urllib.error.HTTPError) as ei:
|
||||
urllib.request.urlopen(req, timeout=10)
|
||||
assert ei.value.code == 500
|
||||
body = json.loads(ei.value.read())
|
||||
assert "no such column: kind" in body["error"]
|
||||
# /health does not touch the DB and still answers.
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{httpd.server_address[1]}/health", timeout=10) as r:
|
||||
assert r.status == 200
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
|
||||
Reference in New Issue
Block a user