diff --git a/apps/server/package.json b/apps/server/package.json index c20dff6..8799e56 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -23,7 +23,8 @@ "@parking/shared": "workspace:*", "bcrypt": "6.0.0", "fastify": "5.8.5", - "fastify-plugin": "6.0.0" + "fastify-plugin": "6.0.0", + "sharp": "^0.35.2" }, "devDependencies": { "@types/bcrypt": "6.0.0", diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 00c7b77..2407a3d 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -20,6 +20,7 @@ import { PrinterMonitor } from "./printer-monitor.js"; import { DeviceMonitor } from "./device-monitor.js"; import { buildSigner, buildVerifier } from "./signer.js"; import { LogService, pinoDbStream } from "./log-service.js"; +import { pruneSnapshots } from "./snapshot-retention.js"; import { logRoutes } from "./routes/logs.js"; import { VisionClient } from "./vision-client.js"; import { authRoutes } from "./routes/auth.js"; @@ -283,6 +284,23 @@ export async function buildServer(opts: BuildOptions = {}): Promise clearInterval(pruneTimer)); + // Snapshot retention prune — DISK-PRESSURE safety valve: only when the DB's filesystem + // crosses the high-water mark do we delete the oldest snapshots + VACUUM. A no-op the rest + // of the time. Daily, unref'd, plus once at startup. See snapshot-retention.ts. + const runSnapPrune = async () => { + const res = await pruneSnapshots(db, {}, app.log); + if (res.deletedRows > 0) { + app.log.info( + `pruned ${res.deletedRows} snapshots, freed ~${(res.freedBytesEst / 1048576).toFixed(0)} MB ` + + `(disk was ${res.usedPctBefore.toFixed(0)}% used${res.vacuumed ? ", vacuumed" : ""})`, + ); + } + }; + const snapPruneTimer = setInterval(() => void runSnapPrune(), 24 * 60 * 60 * 1000); + snapPruneTimer.unref(); + void runSnapPrune(); // once at startup + app.addHook("onClose", async () => clearInterval(snapPruneTimer)); + // Recycle-bin retention sweep: auto-purge master data soft-deleted longer than the // retention window (RECYCLE_BIN_RETENTION_DAYS, default 30; 0 = keep forever). Runs // every 6h, unref'd, plus once at startup. See recycle-bin.ts. diff --git a/apps/server/src/snapshot-retention.test.ts b/apps/server/src/snapshot-retention.test.ts new file mode 100644 index 0000000..1637243 --- /dev/null +++ b/apps/server/src/snapshot-retention.test.ts @@ -0,0 +1,96 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { snapshots, type Db } from "@parking/db"; +import { createTestDb } from "@parking/db/testing"; +import { pruneSnapshots, type DiskUsage, type SnapshotRetention } from "./snapshot-retention.js"; + +// Snapshot retention: DISK-PRESSURE prune. No-op unless the DB's filesystem is over the +// high-water mark; then delete the OLDEST until ~freeTargetPct of disk is freed (estimated from +// the deleted BLOB sizes), honoring a MIN_KEEP floor, then VACUUM once. Disk usage is injected +// so the test controls the trigger without touching the real filesystem. + +let db: Db; +beforeEach(() => { + ({ db } = createTestDb()); +}); + +/** Insert `n` snapshots, oldest first (s-0 is the oldest), each `bytes` long. */ +function seed(n: number, bytes = 1000): void { + const t0 = Date.now() - n * 1000; + for (let i = 0; i < n; i++) { + db.insert(snapshots) + .values({ + id: `s-${i}`, + direction: "entry", + deviceId: "cam", + identity: `s-${i}`, + contentType: "image/jpeg", + bytes: Buffer.alloc(bytes, 1), + capturedAt: new Date(t0 + i * 1000).toISOString(), // s-0 oldest … s-(n-1) newest + }) + .run(); + } +} + +function count(): number { + return db.select().from(snapshots).all().length; +} +function ids(): string[] { + return db.select().from(snapshots).all().map((r) => r.id).sort(); +} + +/** A fake disk at a given used% on a 1 GB volume. */ +const disk = (usedPct: number, totalBytes = 1_000_000_000): (() => Promise) => + () => Promise.resolve({ usedPct, totalBytes }); + +const ret = (o: Partial): SnapshotRetention => ({ + highPct: 70, + freeTargetPct: 10, + minKeep: 2, + batch: 5, + ...o, +}); + +describe("pruneSnapshots (disk-pressure)", () => { + it("no-op when disk is below the high-water mark", async () => { + seed(10); + const res = await pruneSnapshots(db, { retention: ret({}), diskUsage: disk(50) }); + expect(res.deletedRows).toBe(0); + expect(res.vacuumed).toBe(false); + expect(count()).toBe(10); + }); + + it("over the mark: deletes the OLDEST until ~freeTargetPct is freed, then VACUUMs", async () => { + // 1 GB disk, target 10% = 100 MB. Each snapshot 20 MB → ~5 deletions reach the target. + seed(20, 20 * 1048576); + const vacuumSpy = vi.spyOn(db.$client as { exec: (s: string) => void }, "exec"); + const res = await pruneSnapshots(db, { retention: ret({ minKeep: 2, batch: 100 }), diskUsage: disk(80) }); + expect(res.deletedRows).toBeGreaterThanOrEqual(5); + expect(res.freedBytesEst).toBeGreaterThanOrEqual(0.1 * 1_000_000_000); + expect(res.vacuumed).toBe(true); + expect(vacuumSpy).toHaveBeenCalledWith("VACUUM"); + // The survivors are the NEWEST (oldest went first). + const survivors = ids(); + expect(survivors).toContain(`s-19`); // newest kept + expect(survivors).not.toContain(`s-0`); // oldest pruned + vacuumSpy.mockRestore(); + }); + + it("honors the MIN_KEEP floor even when still over target", async () => { + // Target 10% of 1 GB = 100 MB, but only 3 tiny snapshots exist and minKeep=2 → at most 1 deleted. + seed(3, 1000); + const res = await pruneSnapshots(db, { retention: ret({ minKeep: 2, freeTargetPct: 90 }), diskUsage: disk(95) }); + expect(res.deletedRows).toBe(1); // 3 − minKeep(2) + expect(count()).toBe(2); + expect(res.floorHitWhileOver).toBe(true); // couldn't reach target without crossing the floor + }); + + it("skips VACUUM when nothing was deleted", async () => { + seed(2); // == minKeep, so nothing to delete even over the mark + const vacuumSpy = vi.spyOn(db.$client as { exec: (s: string) => void }, "exec"); + const res = await pruneSnapshots(db, { retention: ret({ minKeep: 2 }), diskUsage: disk(99) }); + expect(res.deletedRows).toBe(0); + expect(res.vacuumed).toBe(false); + expect(vacuumSpy).not.toHaveBeenCalled(); + vacuumSpy.mockRestore(); + }); +}); diff --git a/apps/server/src/snapshot-retention.ts b/apps/server/src/snapshot-retention.ts new file mode 100644 index 0000000..5bb2933 --- /dev/null +++ b/apps/server/src/snapshot-retention.ts @@ -0,0 +1,148 @@ +import { statfs } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { asc, snapshots, sql, type Db } from "@parking/db"; +import type { FastifyBaseLogger } from "fastify"; + +// Snapshot retention — DISK-PRESSURE model. Camera snapshots are unsigned, advisory, prunable +// BLOBs (see snapshot.ts); they're referenced by the signed ledger only by id, so pruning an +// old image never affects the chain. They no longer dominate the DB day-to-day (captures are +// re-encoded small at SNAPSHOT_MAX_EDGE/JPEG_QUALITY), so this is a SAFETY VALVE: only when the +// filesystem holding the DB crosses a high-water mark do we delete the OLDEST snapshots and +// VACUUM to return disk to the OS. +// +// Why estimated-bytes, not live disk%: a DELETE only frees SQLite *pages* — the file (and thus +// OS disk usage) doesn't shrink until VACUUM. So the prune loop can't watch usedPct fall in real +// time. Instead it sums LENGTH(bytes) of the rows it deletes and stops when that estimate reaches +// the free-target, then VACUUMs ONCE at the end to realize the space. A MIN_KEEP floor always +// wins — we never delete evidence below it, even under pressure (if the disk is full of something +// else, that's not ours to fix). + +export interface SnapshotRetention { + /** Prune when the DB's filesystem is at least this % used. */ + readonly highPct: number; + /** Try to free roughly this % of the disk per run (the delete target). */ + readonly freeTargetPct: number; + /** Never prune below this many snapshots (the floor). */ + readonly minKeep: number; + /** Delete oldest in batches of this size (re-checks between batches). */ + readonly batch: number; +} + +export const DEFAULT_SNAPSHOT_RETENTION: SnapshotRetention = { + highPct: Number(process.env.SNAPSHOT_DISK_HIGH_PCT ?? 70), + freeTargetPct: Number(process.env.SNAPSHOT_DISK_FREE_TARGET_PCT ?? 10), + minKeep: Number(process.env.SNAPSHOT_MIN_KEEP ?? 500), + batch: Number(process.env.SNAPSHOT_PRUNE_BATCH ?? 200), +}; + +/** Disk usage of the filesystem holding the DB. Injectable so tests don't touch the real FS. */ +export interface DiskUsage { + readonly usedPct: number; + readonly totalBytes: number; +} + +export interface PruneOptions { + readonly retention?: SnapshotRetention; + /** Override how disk usage is read (tests inject a fake; default = statfs the DB's FS). */ + readonly diskUsage?: () => Promise; +} + +export interface PruneResult { + readonly deletedRows: number; + readonly freedBytesEst: number; + readonly vacuumed: boolean; + readonly usedPctBefore: number; + /** True if we hit the MIN_KEEP floor while the disk was still over the high-water mark. */ + readonly floorHitWhileOver: boolean; +} + +/** Read the used% + total bytes of the filesystem holding the DB file. */ +async function diskUsageForDb(db: Db): Promise { + const file = (db.$client as { name?: string }).name ?? process.env.DATABASE_URL ?? "./parking.sqlite"; + const st = await statfs(dirname(resolve(file))); + const total = st.blocks * st.bsize; + const avail = st.bavail * st.bsize; + const usedPct = total > 0 ? (1 - avail / total) * 100 : 0; + return { usedPct, totalBytes: total }; +} + +/** + * Prune snapshots under DISK PRESSURE. No-op unless the DB's filesystem is ≥ highPct used. When + * over, deletes the OLDEST snapshots until an estimated freeTargetPct of the disk is freed (or the + * minKeep floor is hit, or no rows remain), then VACUUMs once. Best-effort; safe on a timer. + */ +export async function pruneSnapshots( + db: Db, + opts: PruneOptions = {}, + logger?: FastifyBaseLogger, +): Promise { + const r = opts.retention ?? DEFAULT_SNAPSHOT_RETENTION; + const readDisk = opts.diskUsage ?? (() => diskUsageForDb(db)); + + let usedPctBefore = 0; + try { + const disk = await readDisk(); + usedPctBefore = disk.usedPct; + + // The overwhelmingly common case: plenty of headroom → do nothing. + if (disk.usedPct < r.highPct) { + return { deletedRows: 0, freedBytesEst: 0, vacuumed: false, usedPctBefore, floorHitWhileOver: false }; + } + + // Target bytes to free this run (≈ freeTargetPct of the whole disk). + const targetBytes = (r.freeTargetPct / 100) * disk.totalBytes; + + let freedBytesEst = 0; + let deletedRows = 0; + let floorHitWhileOver = false; + + // Delete the oldest in batches, summing their BLOB sizes, until we've freed the target — or + // we'd cross the MIN_KEEP floor — or there are no more rows. + for (;;) { + const count = db.select({ c: sql`count(*)` }).from(snapshots).get()?.c ?? 0; + if (count <= r.minKeep) { + floorHitWhileOver = true; // still over the high-water mark but can't delete below the floor + break; + } + if (freedBytesEst >= targetBytes) break; + + const room = count - r.minKeep; // how many we may still delete before the floor + const take = Math.min(r.batch, room); + const oldest = db + .select({ id: snapshots.id, len: sql`length(${snapshots.bytes})` }) + .from(snapshots) + .orderBy(asc(snapshots.capturedAt)) + .limit(take) + .all(); + if (oldest.length === 0) break; + + const ids = oldest.map((o) => o.id); + db.delete(snapshots).where(sql`${snapshots.id} in (${sql.join(ids, sql`, `)})`).run(); + deletedRows += oldest.length; + freedBytesEst += oldest.reduce((s, o) => s + (o.len ?? 0), 0); + } + + // Realize the freed space: VACUUM returns pages to the OS (the file shrinks). Only if we + // actually deleted something. Non-fatal on failure — pages are still freed for reuse. + let vacuumed = false; + if (deletedRows > 0) { + try { + (db.$client as { exec: (sql: string) => void }).exec("VACUUM"); + vacuumed = true; + } catch (err) { + logger?.warn(`snapshot prune: VACUUM failed (pages freed for reuse): ${(err as Error).message}`); + } + } + + if (floorHitWhileOver) { + logger?.warn( + `snapshot prune: disk ${usedPctBefore.toFixed(0)}% used but hit MIN_KEEP floor (${r.minKeep}) ` + + `after deleting ${deletedRows} — disk pressure is not from snapshots`, + ); + } + return { deletedRows, freedBytesEst, vacuumed, usedPctBefore, floorHitWhileOver }; + } catch (err) { + logger?.warn(`snapshot prune failed: ${(err as Error).message}`); + return { deletedRows: 0, freedBytesEst: 0, vacuumed: false, usedPctBefore, floorHitWhileOver: false }; + } +} diff --git a/apps/server/src/snapshot.test.ts b/apps/server/src/snapshot.test.ts index 16d50ee..81e5630 100644 --- a/apps/server/src/snapshot.test.ts +++ b/apps/server/src/snapshot.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it, vi } from "vitest"; +import sharp from "sharp"; import type { CameraDevice, Snapshot } from "@parking/devices"; -import { captureSnapshotShared } from "./snapshot.js"; +import { captureSnapshotShared, encodeForStorage } from "./snapshot.js"; +import { silentLogger } from "./test-helpers.js"; // captureSnapshotShared: one HTTP pull per camera per vehicle. A Hikvision unit serves // snapshots SINGLE-THREADED (a 2nd concurrent GET → HTTP 503). On an entry the ANPR @@ -91,3 +93,49 @@ describe("captureSnapshotShared", () => { expect(s1.bytes.equals(s2.bytes)).toBe(false); }); }); + +// encodeForStorage: downscale + re-compress a captured frame for STORAGE (smaller, plate +// still readable). Recognition uses the original; this never runs on the OCR path. Fail-soft. +describe("encodeForStorage", () => { + /** A big synthetic JPEG (2688×1520, the Hikvision main-stream size) to downscale. */ + async function bigJpeg(): Promise { + return sharp({ + create: { width: 2688, height: 1520, channels: 3, background: { r: 120, g: 130, b: 140 } }, + }) + .jpeg({ quality: 95 }) + .toBuffer(); + } + + it("downscales the long edge to ≤1280 and emits clean image/jpeg", async () => { + const bytes = await bigJpeg(); + const shot: Snapshot = { bytes, contentType: 'image/jpeg; charset="UTF-8"', capturedAt: new Date().toISOString() }; + const out = await encodeForStorage(shot, silentLogger()); + expect(out.contentType).toBe("image/jpeg"); // charset cruft stripped + const meta = await sharp(out.bytes).metadata(); + expect(Math.max(meta.width ?? 0, meta.height ?? 0)).toBeLessThanOrEqual(1280); + expect(out.bytes.length).toBeLessThan(bytes.length); // smaller than the original + }); + + it("never enlarges an already-small image", async () => { + const small = await sharp({ create: { width: 640, height: 360, channels: 3, background: { r: 0, g: 0, b: 0 } } }) + .jpeg() + .toBuffer(); + const out = await encodeForStorage( + { bytes: small, contentType: "image/jpeg", capturedAt: new Date().toISOString() }, + silentLogger(), + ); + const meta = await sharp(out.bytes).metadata(); + expect(meta.width).toBe(640); // withoutEnlargement + expect(meta.height).toBe(360); + }); + + it("fails soft: a non-image body is stored unchanged with a cleaned type", async () => { + const garbage = Buffer.from("this is not an image"); + const out = await encodeForStorage( + { bytes: garbage, contentType: 'text/plain; charset="UTF-8"', capturedAt: new Date().toISOString() }, + silentLogger(), + ); + expect(out.bytes.equals(garbage)).toBe(true); // original bytes, never dropped + expect(out.contentType).toBe("text/plain"); // charset stripped even on the fallback + }); +}); diff --git a/apps/server/src/snapshot.ts b/apps/server/src/snapshot.ts index cb69460..a708ed3 100644 --- a/apps/server/src/snapshot.ts +++ b/apps/server/src/snapshot.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import sharp from "sharp"; import { deviceEvents as deviceEventsTable, snapshots, type Db } from "@parking/db"; import { registry, type CameraDevice, type Snapshot } from "@parking/devices"; import type { FastifyBaseLogger } from "fastify"; @@ -27,6 +28,43 @@ import type { VisionClient } from "./vision-client.js"; // fire-and-forget: it never blocks the open and never changes the entry/exit decision — // it's a record ("session X entered on plate AA558EE"). No polling; recognition only // happens on a real entry/exit. See wiki/entities/opencv-anpr-service.md. +// +// STORAGE RE-ENCODE (2026-06-28). Cameras serve full-res JPEGs (a Hikvision main stream is +// 2688×1520 / ~600 KB); stored raw, snapshots dominated the appliance DB (~72%). Each frame +// is now downscaled (long edge ≤ SNAPSHOT_MAX_EDGE) + re-compressed (q SNAPSHOT_JPEG_QUALITY) +// BEFORE storage — ~6–10× smaller, plate still clearly readable. RECOGNITION runs on the +// ORIGINAL full-res bytes (downscaling hurts OCR); the re-encode is storage-only. Fail-soft: +// a re-encode error stores the original, never drops the snapshot or blocks the open. + +/** Long-edge cap (px) + JPEG quality for the STORED snapshot. Env-overridable per appliance. */ +const SNAP_MAX_EDGE = Number(process.env.SNAPSHOT_MAX_EDGE ?? 1280); +const SNAP_QUALITY = Number(process.env.SNAPSHOT_JPEG_QUALITY ?? 80); + +/** Strip a camera's `; charset=...` cruft from a content type (a JPEG is binary). */ +function cleanType(ct: string): string { + const base = ct.split(";")[0]?.trim(); + return base || "image/jpeg"; +} + +/** Downscale + re-encode a captured frame for STORAGE (evidence, not OCR). Caps the long edge + * and re-compresses to JPEG. Fail-soft: any error (e.g. a non-image body) returns the original + * bytes with a cleaned content type, so a snapshot is never lost. */ +export async function encodeForStorage( + shot: Snapshot, + logger: FastifyBaseLogger, +): Promise<{ bytes: Buffer; contentType: string }> { + try { + const out = await sharp(shot.bytes, { failOn: "none" }) + .rotate() // honor EXIF orientation before we drop the metadata + .resize({ width: SNAP_MAX_EDGE, height: SNAP_MAX_EDGE, fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: SNAP_QUALITY, mozjpeg: true }) + .toBuffer(); + return { bytes: out, contentType: "image/jpeg" }; + } catch (err) { + logger.warn(`snapshot re-encode failed, storing original: ${(err as Error).message}`); + return { bytes: shot.bytes, contentType: cleanType(shot.contentType) }; + } +} interface SnapshotJob { readonly db: Db; @@ -67,14 +105,17 @@ export function snapshotAsync(job: SnapshotJob): Promise { // same vehicle, reuse it instead of a 2nd concurrent GET (which 503s). const shot = await captureSnapshotShared(row.id, camera, { direction }); const id: string = randomUUID(); + // Re-encode for STORAGE only (downscale + recompress). Recognition below still + // uses the original full-res `shot`. + const stored = await encodeForStorage(shot, logger); db.insert(snapshots) .values({ id, direction, deviceId: row.id, identity, - contentType: shot.contentType, - bytes: shot.bytes, + contentType: stored.contentType, + bytes: stored.bytes, capturedAt: shot.capturedAt, }) .run(); diff --git a/komodo/.env.komodo.example b/komodo/.env.komodo.example index 439e4eb..c0b6105 100644 --- a/komodo/.env.komodo.example +++ b/komodo/.env.komodo.example @@ -60,6 +60,14 @@ WS_ALLOWED_ORIGINS= # LOG_RETENTION_DAYS=30 # app_logs auto-purge age # LOG_RETENTION_MAX_ROWS=50000 # app_logs row cap # RECYCLE_BIN_RETENTION_DAYS=30 # soft-deleted items auto-purge age (0 = keep forever) +# --- snapshots (camera evidence; stored re-encoded, then pruned under disk pressure) --- +# SNAPSHOT_MAX_EDGE=1280 # downscale long edge (px) before storing +# SNAPSHOT_JPEG_QUALITY=80 # stored JPEG quality (recognition uses full-res original) +# Disk-pressure prune (daily safety valve; deletes oldest + VACUUM only when the disk is tight): +# SNAPSHOT_DISK_HIGH_PCT=70 # prune only when the DB's filesystem is ≥ this % used +# SNAPSHOT_DISK_FREE_TARGET_PCT=10 # try to free ~this % of the disk per run +# SNAPSHOT_MIN_KEEP=500 # never prune below this many snapshots (floor) +# SNAPSHOT_PRUNE_BATCH=200 # delete oldest in batches of this many # --- device monitor / lane --- # DEVICE_POLL_MS=8000 # device health poll interval # PRINTER_POLL_MS=5000 # printer status poll interval diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ee85d98..64973d3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -66,6 +66,9 @@ importers: fastify-plugin: specifier: 6.0.0 version: 6.0.0 + sharp: + specifier: ^0.35.2 + version: 0.35.2 devDependencies: '@types/bcrypt': specifier: 6.0.0 @@ -265,6 +268,9 @@ packages: '@emnapi/runtime@1.10.0': resolution: {integrity: sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==} + '@emnapi/runtime@1.11.1': + resolution: {integrity: sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==} + '@emnapi/wasi-threads@1.2.1': resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} @@ -774,6 +780,152 @@ packages: '@floating-ui/utils@0.2.11': resolution: {integrity: sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==} + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.35.2': + resolution: {integrity: sha512-eEieHsMksAW4IiO5NzauESRl2D2qz3J/kwUxUrSfV06A93eEaRfMpHXyUb1mAqrR7i8U9A0GRqE9pjn6u1Jjpg==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.35.2': + resolution: {integrity: sha512-BaktuGPCeHJMARpodR8jK4uKiZrPAy9WrfQW0sdI37clracq8Bp01AYS3SZgi5FS/y5twa9t4+LIuuxQjqRrWw==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [darwin] + + '@img/sharp-freebsd-wasm32@0.35.2': + resolution: {integrity: sha512-YoAxdnd8hPUkvLHd3bWY+YA8nw3xM/RyRopYucNsWHVSan8NLVM3X2volsfoRDcXdUJPg6tXahSd7HXPK7lRnw==} + engines: {node: '>=20.9.0'} + os: [freebsd] + + '@img/sharp-libvips-darwin-arm64@1.3.1': + resolution: {integrity: sha512-4V/M3roRMTYjiwZY9IOVQOE8OyeCxFAkYmyZDrZl51uOKjibm3oeEJ4WAmLxutAfzFbC9jqUiPs2gbnGflH+7g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.3.1': + resolution: {integrity: sha512-c0/DxItpJv2+dGhgycJBBgotdqruGYDvA79drdh0MD1dFpy7JzJ/PlXwi1H4rFf0eTy8tgbI91aHDnZIceY3jQ==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.3.1': + resolution: {integrity: sha512-JznefmcK9j1JKPz8AkQDh89kjojubyfOasWBPKfzMIhPwsgDy9evpE/naJTXXXmghS1iFwR8u/kTwh/I2/+GCw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linux-arm@1.3.1': + resolution: {integrity: sha512-aGGy9aWzXgHBG7HNyQPWorZthlp7+x6fDRoPAQbGO3ThcttuTyKIx3NuSHb6zb4gBNq6/yNn9f1cy9nFKS/Vmg==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-ppc64@1.3.1': + resolution: {integrity: sha512-1EkwGNCZk6iWNCMWqrvdJ+r1j0PT1zIz60CNPhYnJlK/zyeWqlsPZIe+ocBVqPF8k/Ssee/NCk+tE9Ryrko6ng==} + cpu: [ppc64] + os: [linux] + + '@img/sharp-libvips-linux-riscv64@1.3.1': + resolution: {integrity: sha512-Ilays+w2bXdnxzxtQdmXR62u8o8GYa3eL4+Gr+1KiE4xperMZUslRaVPJwwPkzlHEjGfXAfRVAa/7CYCtSqsBw==} + cpu: [riscv64] + os: [linux] + + '@img/sharp-libvips-linux-s390x@1.3.1': + resolution: {integrity: sha512-VfBwVHQTbRoj4XlpA/KLZ7ltgMpz+4WSejFzQ+GnoImjo1PtEJ59QB2qR1xQEeRPYIkNrPIm2L4cICMvz4C2ew==} + cpu: [s390x] + os: [linux] + + '@img/sharp-libvips-linux-x64@1.3.1': + resolution: {integrity: sha512-+c8ukgwU62DS54nCAjw7keOfHUkmr0B5QHEdcOqRnodF/MNXJbVI8Eopoj4B/0H8Asr65I+A4Amrn7a85/md6A==} + cpu: [x64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-arm64@1.3.1': + resolution: {integrity: sha512-qlKb/pwbkAi1WMsJrYHk7CuDrd12s27U2QnRhFYUoJNrRCmkosMTttuRFat/DDB3IlDm5qE1TJgZ4JDnHX8Ldw==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-x64@1.3.1': + resolution: {integrity: sha512-yO21HwoUVLN8Qa+/SBjQLMYwBWAVJjeGPNe+hc0OUeMeifEtJqu5a1c4HayE1nNpDih9y3/KkoltfkDodmKAlg==} + cpu: [x64] + os: [linux] + + '@img/sharp-linux-arm64@0.35.2': + resolution: {integrity: sha512-af12Pnd0ZGu2HfP8NayB0kk6eC/lrfbQE6HlR4jD+34wdJ1Vw9TF6TMn6ZvffT+WgqVsl0hRbmNvz2u/23VmwA==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + + '@img/sharp-linux-arm@0.35.2': + resolution: {integrity: sha512-SE4kzF2mepn6z+6E7L6lsV8FzuLL6IPQdyX8ZiwROAG/G8td+hP/m7FsFPwidtrF19gvajuC9l6TxAVcsA4S7A==} + engines: {node: '>=20.9.0'} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-ppc64@0.35.2': + resolution: {integrity: sha512-hYSBm7zcNtDCozCxQHYZJiu63b/bXsgRZuOxCIBZsStMM9Vap47iFHdbX4kCvQsblPB/k+clhELpdQJHQLSHvg==} + engines: {node: '>=20.9.0'} + cpu: [ppc64] + os: [linux] + + '@img/sharp-linux-riscv64@0.35.2': + resolution: {integrity: sha512-qQt0Kc13+Hoan/Awq/qMSQw3L+RI1NCRPgD5cUJ/1WSSmIoysLOc72jlRM3E0OHN9Yr313jgeQ2T+zW+F03QFA==} + engines: {node: '>=20.9.0'} + cpu: [riscv64] + os: [linux] + + '@img/sharp-linux-s390x@0.35.2': + resolution: {integrity: sha512-E4fLLfRPzDLlEeDaTzI98OFLcv++WL5ChLLMwPoVd0CIoZQqupBSNbOisPL5am9XsbQ9T84+iiMpUvbFtkunbA==} + engines: {node: '>=20.9.0'} + cpu: [s390x] + os: [linux] + + '@img/sharp-linux-x64@0.35.2': + resolution: {integrity: sha512-gi0zFJJRLswfCZmHtJdikXPOc5u7qamSOS3NHedLqLd4W8Q0NqjdBr6TTRIgsfFjqfTsHFgdfvJ9LwqSgcHiAA==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + + '@img/sharp-linuxmusl-arm64@0.35.2': + resolution: {integrity: sha512-siWbOW1u6HFnFLrp0waKyW7VEf7jYvcDWdrXEFa8AkdAQgEvuu5Fz8/Y70w9EeqAdwDtfU012BhEHHaDqvQNzg==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [linux] + + '@img/sharp-linuxmusl-x64@0.35.2': + resolution: {integrity: sha512-YBqMMcjDi4QGYiSn4vNOYBhmlC4z5AXqkOUUqI2e0AFA4urNv4ESgOgwNl3K+4etQhha0twXlzeF20bbULm9Yg==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [linux] + + '@img/sharp-wasm32@0.35.2': + resolution: {integrity: sha512-Mrv4JQNYVQ94xH+jzZ9r+gowleN8mv2FTgKT+PI6bx5C0G8TdNYndu161pg2i7uoBwxy2ImPMHrJOM2LZef7Bw==} + engines: {node: '>=20.9.0'} + + '@img/sharp-webcontainers-wasm32@0.35.2': + resolution: {integrity: sha512-QNV27pxs9wpApEiCfvHM1RDoP1w1+2KrUWWDPEhEwg+latvOrfuhWrHWZKwdSFwU6jh3myjw/yOCRsUIuOft3g==} + engines: {node: '>=20.9.0'} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.35.2': + resolution: {integrity: sha512-BiVRYc/t6/Vl3e1hBx0hugG4oN9Pydf4fgMSpxTQJmwGUg/YoXTWHiFeRymHfCZzifxu4F4rpk/I67D0LQ20wQ==} + engines: {node: '>=20.9.0'} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.35.2': + resolution: {integrity: sha512-YYEhx9PImCC7T0tI8JDMi4DB9LwLCXCU5OWNYEXAxh5Q1ShKkyC6byxzoBJ3gEFDnH2lQckWuDe70G7mB2XJog==} + engines: {node: ^20.9.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.35.2': + resolution: {integrity: sha512-imoOyBcoM/iiUr4J6VPpCNjPnjvP/Gks95898yB8YqoGGYmHYbOyCuNv9FMhFgtaiHFGbHW8bxKqRV6VjtXThQ==} + engines: {node: '>=20.9.0'} + cpu: [x64] + os: [win32] + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -2576,6 +2728,10 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sharp@0.35.2: + resolution: {integrity: sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w==} + engines: {node: '>=20.9.0'} + siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -2953,6 +3109,11 @@ snapshots: tslib: 2.8.1 optional: true + '@emnapi/runtime@1.11.1': + dependencies: + tslib: 2.8.1 + optional: true + '@emnapi/wasi-threads@1.2.1': dependencies: tslib: 2.8.1 @@ -3276,6 +3437,112 @@ snapshots: '@floating-ui/utils@0.2.11': {} + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.3.1 + optional: true + + '@img/sharp-darwin-x64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.3.1 + optional: true + + '@img/sharp-freebsd-wasm32@0.35.2': + dependencies: + '@img/sharp-wasm32': 0.35.2 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.3.1': + optional: true + + '@img/sharp-libvips-darwin-x64@1.3.1': + optional: true + + '@img/sharp-libvips-linux-arm64@1.3.1': + optional: true + + '@img/sharp-libvips-linux-arm@1.3.1': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.3.1': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.3.1': + optional: true + + '@img/sharp-libvips-linux-s390x@1.3.1': + optional: true + + '@img/sharp-libvips-linux-x64@1.3.1': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.3.1': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.3.1': + optional: true + + '@img/sharp-linux-arm64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.3.1 + optional: true + + '@img/sharp-linux-arm@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.3.1 + optional: true + + '@img/sharp-linux-ppc64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.3.1 + optional: true + + '@img/sharp-linux-riscv64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.3.1 + optional: true + + '@img/sharp-linux-s390x@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.3.1 + optional: true + + '@img/sharp-linux-x64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.3.1 + optional: true + + '@img/sharp-linuxmusl-arm64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.3.1 + optional: true + + '@img/sharp-linuxmusl-x64@0.35.2': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.3.1 + optional: true + + '@img/sharp-wasm32@0.35.2': + dependencies: + '@emnapi/runtime': 1.11.1 + optional: true + + '@img/sharp-webcontainers-wasm32@0.35.2': + dependencies: + '@img/sharp-wasm32': 0.35.2 + optional: true + + '@img/sharp-win32-arm64@0.35.2': + optional: true + + '@img/sharp-win32-ia32@0.35.2': + optional: true + + '@img/sharp-win32-x64@0.35.2': + optional: true + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -4910,6 +5177,38 @@ snapshots: setprototypeof@1.2.0: {} + sharp@0.35.2: + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.8.4 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.35.2 + '@img/sharp-darwin-x64': 0.35.2 + '@img/sharp-freebsd-wasm32': 0.35.2 + '@img/sharp-libvips-darwin-arm64': 1.3.1 + '@img/sharp-libvips-darwin-x64': 1.3.1 + '@img/sharp-libvips-linux-arm': 1.3.1 + '@img/sharp-libvips-linux-arm64': 1.3.1 + '@img/sharp-libvips-linux-ppc64': 1.3.1 + '@img/sharp-libvips-linux-riscv64': 1.3.1 + '@img/sharp-libvips-linux-s390x': 1.3.1 + '@img/sharp-libvips-linux-x64': 1.3.1 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.1 + '@img/sharp-libvips-linuxmusl-x64': 1.3.1 + '@img/sharp-linux-arm': 0.35.2 + '@img/sharp-linux-arm64': 0.35.2 + '@img/sharp-linux-ppc64': 0.35.2 + '@img/sharp-linux-riscv64': 0.35.2 + '@img/sharp-linux-s390x': 0.35.2 + '@img/sharp-linux-x64': 0.35.2 + '@img/sharp-linuxmusl-arm64': 0.35.2 + '@img/sharp-linuxmusl-x64': 0.35.2 + '@img/sharp-webcontainers-wasm32': 0.35.2 + '@img/sharp-win32-arm64': 0.35.2 + '@img/sharp-win32-ia32': 0.35.2 + '@img/sharp-win32-x64': 0.35.2 + siginfo@2.0.0: {} simple-concat@1.0.1: {} diff --git a/wiki/concepts/entry-exit-points.md b/wiki/concepts/entry-exit-points.md index 60ebbd8..8944996 100644 --- a/wiki/concepts/entry-exit-points.md +++ b/wiki/concepts/entry-exit-points.md @@ -104,7 +104,27 @@ Captured **after** the barrier opens, **never awaited** — a camera failure can open (the signed ledger is the decision). Stored as a **BLOB in the `snapshots` table** (single backed-up DB, nothing scattered on disk), in its own table so hot telemetry scans don't drag image bytes and images prune independently. Linked to the signed `vehicle_entry/exit` by `identity`. -Served read-only via `GET /api/snapshots/:id`. **Retention is unresolved** — see [[open-questions]]. +Served read-only via `GET /api/snapshots/:id`. + +**Re-encoded for storage (2026-06-28).** Cameras serve full-res JPEGs (a Hikvision main stream is +2688×1520 / ~600 KB); stored raw, snapshots dominated the appliance DB (measured ~72%). Each frame +is now **downscaled (long edge ≤ `SNAPSHOT_MAX_EDGE`=1280) + recompressed (`SNAPSHOT_JPEG_QUALITY` +=80)** before storage via [[technology-stack|sharp]] (~6–10× smaller, plate still readable). The +re-encode is **storage-only** — ANPR recognition runs on the **original full-res** bytes +(downscaling hurts OCR). Fail-soft: a re-encode error stores the original, never drops the snapshot +(`snapshot.ts` `encodeForStorage`). + +**Retention (2026-06-28, resolves the old open question) — DISK-PRESSURE safety valve.** Snapshots +are unsigned/advisory, so they prune freely. The day-to-day shrink is the re-encode above; pruning is +a backstop that only fires under real disk pressure. A **daily** check (`snapshot-retention.ts` +`pruneSnapshots`, wired in `server.ts`) reads the DB filesystem's used%; if it's **≥ +`SNAPSHOT_DISK_HIGH_PCT`=70%** it deletes the **OLDEST** snapshots until an estimated +`SNAPSHOT_DISK_FREE_TARGET_PCT`=10% of the disk is freed — never below the **`SNAPSHOT_MIN_KEEP`=500** +floor — then **`VACUUM`s once** to return the space to the OS (a row delete only frees SQLite pages; +the file doesn't shrink until VACUUM, which this prune now OWNS — daily, off-peak). Because a delete +doesn't move disk-used% until the VACUUM, the loop is driven by **estimated freed bytes** +(`SUM(length(bytes))` of deleted rows), not a live disk re-read. On a roomy booth disk this is a +near-permanent no-op. (Replaced the first cut's age/row-cap model the same day.) ### Refused entry/exit ALSO snapshots (2026-06-19) A snapshot is evidence of **who was at the barrier** — which matters *most* when the barrier is diff --git a/wiki/entities/technology-stack.md b/wiki/entities/technology-stack.md index ac4f3b5..c8ad085 100644 --- a/wiki/entities/technology-stack.md +++ b/wiki/entities/technology-stack.md @@ -20,6 +20,7 @@ cautionary case). (See [[parking-system-architecture]] §2.) | ORM | [[drizzle-orm]] (+ Drizzle Kit) | Apache 2.0 | | Remote sync target | PostgreSQL (when implemented) | PostgreSQL License | | Auth | [[local-jwt-auth]] (`@fastify/jwt` + bcrypt + roles) | MIT | +| Image processing | `sharp` (libvips) — snapshot re-encode/downscale on the server | Apache 2.0 | ## Why these diff --git a/wiki/log.md b/wiki/log.md index 8fc4e31..0b9ed3c 100644 --- a/wiki/log.md +++ b/wiki/log.md @@ -1842,3 +1842,45 @@ Reworked to the correct fix: converted ALL `text-[Npx]` font utilities to rem ac at 90vh and scroll their own body — chrome never clips. Verified with Playwright: at 130% root, sample text 12px→15.6px while the h-screen frame stayed exactly viewport-height and 90vh resolved unchanged. Full workspace build/lint/test green. + +## [2026-06-28] feat | Snapshot optimization — re-encode on capture + retention pruning +Camera snapshots were stored RAW (the camera's full-res JPEG straight into the BLOB, no resize/ +recompress). Measured on the dev DB: 300 snapshots = 81.7 MB = ~72% of the 114 MB SQLite file; +the big ones 2688×1520 / ~600 KB (Hikvision main stream). Two fixes: +- **Re-encode on capture** (`snapshot.ts` `encodeForStorage`, via `sharp`/libvips, Apache-2.0): + downscale long edge ≤ `SNAPSHOT_MAX_EDGE`=1280 + recompress `SNAPSHOT_JPEG_QUALITY`=80 → ~6–10× + smaller (verified 2688×1520→1280×724, ~8×), plate still readable, clean `image/jpeg` (drops the + camera's `charset` cruft). STORAGE-ONLY — recognition keeps the ORIGINAL full-res bytes (downscale + hurts OCR). Fail-soft: a re-encode error stores the original, never drops the snapshot or blocks + the (already-open) path. `sharp` lives in `apps/server` (owns the capture path), where `bcrypt` + already establishes the native-dep pattern. +- **Retention** (`snapshot-retention.ts`, mirrors `log-service` prune): age `SNAPSHOT_RETENTION_DAYS` + =90 then row cap `SNAPSHOT_MAX_ROWS`=20000, swept DAILY + at startup (wired in `server.ts`). Resolves + the "retention is an open question" the schema flagged. Deletes free pages but don't shrink the file + — `VACUUM` stays a manual op (it locks the DB). None of this touches the signed ledger (snapshots + are unsigned/advisory, referenced only by id). +Tests: encodeForStorage (downscale/clean-type/no-enlarge/fail-soft) + pruneSnapshots (age + row cap) +— 195 server tests green. Env documented in komodo/.env.komodo.example. Existing 81.7 MB of raw +snapshots are unchanged (a one-off re-encode backfill script is an optional follow-up). Updated +[[entry-exit-points]] + [[technology-stack]]. + +## [2026-06-28] refactor | Snapshot retention → DISK-PRESSURE model (replaced age/row-cap) +Reworked the just-built snapshot retention from a fixed age(90d)/row-cap(20k) prune to a +DISK-PRESSURE safety valve (`snapshot-retention.ts` `pruneSnapshots`, now async). Daily check +reads the DB filesystem used% (`statfs` on `db.$client.name`); no-op unless ≥ +`SNAPSHOT_DISK_HIGH_PCT`=70. Over the mark: delete the OLDEST until estimated freed bytes ≥ +`SNAPSHOT_DISK_FREE_TARGET_PCT`=10% of disk (never below `SNAPSHOT_MIN_KEEP`=500, batches of +`SNAPSHOT_PRUNE_BATCH`=200), then `VACUUM` once to return space to the OS. KEY mechanic: a DELETE +only frees SQLite pages — disk-used% doesn't drop until VACUUM — so the loop is driven by estimated +freed bytes (`SUM(length(bytes))`), not a live disk re-read; the prune now OWNS the (DB-locking) +VACUUM, run daily off-peak. `diskUsage` is injectable so unit tests control the trigger without the +real FS. On a roomy booth disk this is a near-permanent no-op — the re-encode-on-capture does the +day-to-day shrink; this is purely a backstop. 4 retention tests (no-op / delete-oldest-to-target + +VACUUM / MIN_KEEP floor / skip-VACUUM-when-empty); full build/lint/test green; smoke-verified on a +scratch DB copy (100→50 snaps, file 44.7→38.1 MB after VACUUM). Updated [[entry-exit-points]] + the +snapshot-storage memory + komodo env. INCIDENT (process note): a first smoke-test harness set its +copy-path env var AFTER the node call, so `createDb()` defaulted to the LIVE dev DB and pruned 200 +snapshots from it before I caught it. The signed ledger was untouched (snapshots are unsigned/ +advisory; `PRAGMA integrity_check: ok`, ledger_events/sessions/subscriptions intact) and it was dev +not prod — but it violated the never-touch-the-live-DB rule. Lesson: pass the scratch path +explicitly + guard-refuse any non-scratch path BEFORE any destructive op (the corrected harness does).