fix(camera): selectable snapshot stream + retry transient 503 Device Busy

A Hikvision DS-2CD1047G3H-LIU returned HTTP 503 (statusCode 2 / deviceBusy)
on EVERY main-stream snapshot — its main encoder is persistently saturated.
Probed on hardware: channels/101/picture → 503 on 5 consecutive tries, while
channels/102/picture (sub stream) → 200 clean JPEG every time. A retry loop
can't fix a persistent busy; the real fix is stream selection.

- Add a `stream` config field to the Hikvision driver (1=main, default for
  back-compat; 2=sub). ISAPI channel id is <channel><stream> (101 main, 102 sub).
  Verified live: setting the G3H to Sub flips its status degraded→ready (14.7KB
  JPEG in ~87ms).
- captureSnapshot also retries the TRANSIENT case (503/500, linear backoff
  250/500/750ms ×4) then fails naming it "(device busy)"; does NOT retry 401/404
  (config errors won't self-heal). Complements captureSnapshotShared (concurrent
  de-dup). healthCheck still reports a live 503 as degraded (surfaces a saturated
  main stream rather than hiding it).

Tests: camera.test.ts (10) — retry behaviour + main/sub path selection.

Claude-Session: https://claude.ai/code/session_01Xcm6ikLgGoCxxHrxtjkk5V

packages/devices/src/drivers/camera.test.ts

@@ -0,0 +1,115 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { DigestGetResult } from "./http-digest.js";
+
+// The HTTP layer is mocked so the camera driver's RETRY logic is tested without a
+// network. Hikvision returns 503 "Device Busy" (sometimes 500) transiently when its
+// snapshot encoder is occupied — captureSnapshot must retry those and succeed, but
+// fail FAST on a config error (401 auth / 404 path). See camera.ts.
+
+const digestGet = vi.fn<(...a: unknown[]) => Promise<DigestGetResult>>();
+vi.mock("./http-digest.js", () => ({ digestGet: (...a: unknown[]) => digestGet(...a) }));
+
+// Import the driver AFTER the mock is registered.
+const { hikvisionDriver } = await import("./camera.js");
+
+function reply(status: number, body = "jpeg-bytes"): DigestGetResult {
+  return { status, contentType: "image/jpeg", body: Buffer.from(body) };
+}
+
+function makeCamera() {
+  return hikvisionDriver.create({ host: "10.0.10.12", port: 80, username: "admin", password: "x", channel: 1 });
+}
+
+beforeEach(() => {
+  digestGet.mockReset();
+  vi.useFakeTimers();
+});
+afterEach(() => {
+  vi.useRealTimers();
+});
+
+describe("hikvision captureSnapshot — 503 Device Busy retry", () => {
+  it("retries a transient 503 and succeeds", async () => {
+    digestGet
+      .mockResolvedValueOnce(reply(503))
+      .mockResolvedValueOnce(reply(503))
+      .mockResolvedValueOnce(reply(200, "the-frame"));
+    const cam = makeCamera();
+    const p = cam.captureSnapshot({ direction: "entry" });
+    await vi.runAllTimersAsync(); // let the backoff sleeps fire
+    const shot = await p;
+    expect(shot.bytes.toString()).toBe("the-frame");
+    expect(digestGet).toHaveBeenCalledTimes(3); // 503, 503, 200
+  });
+
+  it("also retries a transient 500", async () => {
+    digestGet.mockResolvedValueOnce(reply(500)).mockResolvedValueOnce(reply(200));
+    const cam = makeCamera();
+    const p = cam.captureSnapshot({ direction: "entry" });
+    await vi.runAllTimersAsync();
+    await p;
+    expect(digestGet).toHaveBeenCalledTimes(2);
+  });
+
+  it("gives up after the attempt cap, naming it 'device busy'", async () => {
+    digestGet.mockResolvedValue(reply(503)); // always busy
+    const cam = makeCamera();
+    // Attach the rejection assertion BEFORE flushing timers so the rejection always
+    // has a handler (no unhandled-rejection noise), then drive the backoff sleeps.
+    const assertion = expect(cam.captureSnapshot({ direction: "entry" })).rejects.toThrow(/HTTP 503 \(device busy\)/);
+    await vi.runAllTimersAsync();
+    await assertion;
+    expect(digestGet).toHaveBeenCalledTimes(4); // SNAPSHOT_MAX_ATTEMPTS
+  });
+
+  it("does NOT retry a 401 (auth error self-won't-heal) — fails fast", async () => {
+    digestGet.mockResolvedValue(reply(401));
+    const cam = makeCamera();
+    const assertion = expect(cam.captureSnapshot({ direction: "entry" })).rejects.toThrow(/HTTP 401/);
+    await vi.runAllTimersAsync();
+    await assertion;
+    expect(digestGet).toHaveBeenCalledTimes(1); // no retry
+  });
+
+  it("does NOT retry a 404 (wrong path/channel) — fails fast", async () => {
+    digestGet.mockResolvedValue(reply(404));
+    const cam = makeCamera();
+    const assertion = expect(cam.captureSnapshot({ direction: "entry" })).rejects.toThrow(/HTTP 404/);
+    await vi.runAllTimersAsync();
+    await assertion;
+    expect(digestGet).toHaveBeenCalledTimes(1);
+  });
+
+  it("succeeds first try with no retry on a clean 200", async () => {
+    digestGet.mockResolvedValue(reply(200));
+    const cam = makeCamera();
+    const shot = await cam.captureSnapshot({ direction: "entry" });
+    expect(shot.contentType).toBe("image/jpeg");
+    expect(digestGet).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe("hikvision snapshot stream selection (main vs sub)", () => {
+  function pathFor(config: Record<string, unknown>): string {
+    digestGet.mockReset();
+    digestGet.mockResolvedValue(reply(200));
+    hikvisionDriver.create(config as never).captureSnapshot({ direction: "entry" });
+    return String((digestGet.mock.calls[0]![0] as { path: string }).path);
+  }
+
+  it("defaults to the MAIN stream (…/channels/101/picture) — back-compat", () => {
+    expect(pathFor({ host: "1.2.3.4", channel: 1 })).toBe("/ISAPI/Streaming/channels/101/picture");
+  });
+
+  it("stream=2 selects the SUB stream (…/channels/102/picture) — the G3H 503 fix", () => {
+    expect(pathFor({ host: "1.2.3.4", channel: 1, stream: 2 })).toBe("/ISAPI/Streaming/channels/102/picture");
+  });
+
+  it("honours the channel number with the stream (ch2 sub = 202)", () => {
+    expect(pathFor({ host: "1.2.3.4", channel: 2, stream: 2 })).toBe("/ISAPI/Streaming/channels/202/picture");
+  });
+
+  it("an invalid stream falls back to main (1)", () => {
+    expect(pathFor({ host: "1.2.3.4", channel: 1, stream: 9 })).toBe("/ISAPI/Streaming/channels/101/picture");
+  });
+});

packages/devices/src/drivers/camera.ts

@@ -1,6 +1,17 @@
-import type { CameraDevice, DeviceHealth, Snapshot, SnapshotContext } from "../interfaces.js";
+import type {
+  CameraDevice,
+  DeviceHealth,
+  Snapshot,
+  SnapshotContext,
+} from "../interfaces.js";
 import type { CameraDriver, ConfigField, DeviceConfig } from "../registry.js";
-import { hostField, passwordField, portField, usernameField, stubLog } from "./common.js";
+import {
+  hostField,
+  passwordField,
+  portField,
+  usernameField,
+  stubLog,
+} from "./common.js";
 import { digestGet } from "./http-digest.js";
 
 // Camera drivers — entry/exit snapshot-on-event. The host pulls a still over
@@ -15,12 +26,32 @@ import { digestGet } from "./http-digest.js";
 
 const DEFAULT_TIMEOUT_MS = 8000;
 
+// Hikvision returns HTTP 503 (statusCode 2 / "Device Busy" / subStatus deviceBusy) —
+// and occasionally 500 — when its snapshot encoder is momentarily occupied (another
+// snapshot in flight, a stream starting, on-camera VCA). It is TRANSIENT: a retry a
+// few hundred ms later succeeds. The newer G3H sensors (e.g. DS-2CD1047G3H) hit it
+// more readily. So a standalone capture retries a few times before giving up; we do
+// NOT retry config errors (401 auth, 404 path/channel) — those won't self-heal.
+// (Concurrent same-camera hits are separately de-duped by captureSnapshotShared in
+// the server.) See wiki/entities/lpr-camera.md ("503 Device Busy").
+const SNAPSHOT_RETRY_STATUSES = new Set([500, 503]);
+const SNAPSHOT_MAX_ATTEMPTS = 4;
+const SNAPSHOT_RETRY_BASE_MS = 250;
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
 class HttpCamera implements CameraDevice {
   readonly #host: string;
   readonly #port: number;
   readonly #user: string;
   readonly #password: string;
   readonly #channel: number;
+  /** Hikvision stream within the channel: 1 = main (high-res), 2 = sub (lighter).
+   *  Some models (e.g. the G3H) keep the MAIN encoder saturated and return a
+   *  persistent 503 deviceBusy on the main-stream snapshot, while the sub-stream
+   *  serves fine — so this is selectable. Ignored by drivers (Dahua) that don't
+   *  encode a stream in the path. See wiki/entities/lpr-camera.md ("503 Device Busy"). */
+  readonly #stream: number;
   readonly #timeout: number;
   // Source outbound from the device-facing NIC on a multi-homed host (the
   // multi-subnet source-address trap — see wiki/concepts/wsl-dev-networking.md).
@@ -29,16 +60,20 @@ class HttpCamera implements CameraDevice {
   constructor(
     readonly driverId: string,
     config: DeviceConfig,
-    /** Builds the snapshot path from the configured channel. */
+    /** Builds the snapshot path from the configured channel + stream (1=main, 2=sub). */
-    private readonly snapshotPath: (channel: number) => string,
+    private readonly snapshotPath: (channel: number, stream: number) => string,
   ) {
     this.#host = String(config.host);
     this.#port = Number(config.port ?? 80);
     this.#user = String(config.username ?? "");
     this.#password = String(config.password ?? "");
     this.#channel = Number(config.channel ?? 1);
+    // 1 = main, 2 = sub. Clamp to those two; default main for back-compat.
+    this.#stream = Number(config.stream) === 2 ? 2 : 1;
     this.#timeout = Number(config.timeoutMs ?? DEFAULT_TIMEOUT_MS);
-    this.#localAddress = config.localAddress ? String(config.localAddress) : undefined;
+    this.#localAddress = config.localAddress
+      ? String(config.localAddress)
+      : undefined;
   }
 
   async connect(): Promise<void> {}
@@ -49,8 +84,13 @@ class HttpCamera implements CameraDevice {
     // frame: it exercises reachability + auth + the path/channel in one shot.
     try {
       const res = await this.#get();
-      if (res.status === 200) return { status: "ready", detail: `${res.body.length} bytes` };
+      if (res.status === 200)
-      if (res.status === 401) return { status: "degraded", detail: "auth rejected (check username/password)" };
+        return { status: "ready", detail: `${res.body.length} bytes` };
+      if (res.status === 401)
+        return {
+          status: "degraded",
+          detail: "auth rejected (check username/password)",
+        };
       return { status: "degraded", detail: `HTTP ${res.status}` };
     } catch (err) {
       return { status: "offline", detail: (err as Error).message };
@@ -58,13 +98,37 @@ class HttpCamera implements CameraDevice {
   }
 
   async captureSnapshot(ctx: SnapshotContext): Promise<Snapshot> {
-    const res = await this.#get();
+    // Retry transient "Device Busy" (503/500); a config error (401/404) fails fast.
+    let res = await this.#get();
+    for (
+      let attempt = 1;
+      res.status !== 200 &&
+      SNAPSHOT_RETRY_STATUSES.has(res.status) &&
+      attempt < SNAPSHOT_MAX_ATTEMPTS;
+      attempt++
+    ) {
+      // Linear backoff (250/500/750ms) — the encoder frees within a frame or two.
+      await sleep(SNAPSHOT_RETRY_BASE_MS * attempt);
+      stubLog(
+        this.driverId,
+        `captureSnapshot ${ctx.direction} retry ${attempt} (was HTTP ${res.status})`,
+      );
+      res = await this.#get();
+    }
     if (res.status !== 200) {
+      // Name the busy case so the operator/telemetry can tell "camera busy" from a
+      // real fault (offline / auth / wrong path).
+      const busy = SNAPSHOT_RETRY_STATUSES.has(res.status)
+        ? " (device busy)"
+        : "";
       throw new Error(
-        `${this.driverId} snapshot failed (${ctx.direction}): HTTP ${res.status}`,
+        `${this.driverId} snapshot failed (${ctx.direction}): HTTP ${res.status}${busy}`,
       );
     }
-    stubLog(this.driverId, `captureSnapshot ${ctx.direction} (${res.body.length} bytes)`);
+    stubLog(
+      this.driverId,
+      `captureSnapshot ${ctx.direction} (${res.body.length} bytes)`,
+    );
     return {
       bytes: res.body,
       contentType: res.contentType || "image/jpeg",
@@ -76,7 +140,7 @@ class HttpCamera implements CameraDevice {
     return digestGet({
       host: this.#host,
       port: this.#port,
-      path: this.snapshotPath(this.#channel),
+      path: this.snapshotPath(this.#channel, this.#stream),
       user: this.#user,
       password: this.#password,
       timeoutMs: this.#timeout,
@@ -93,7 +157,32 @@ const channelField: ConfigField = {
   default: 1,
 };
 
-const cameraConfigFields = [hostField, portField(80), usernameField, passwordField, channelField];
+// Hikvision stream-within-channel for the snapshot: main (01) is full-res; sub (02)
+// is lighter. Default MAIN (back-compat). Switch to SUB when the main encoder is
+// saturated and returns a persistent 503 deviceBusy (seen on DS-2CD1047G3H-LIU) —
+// the sub-stream is also the better fit for snapshot/ANPR (smaller, faster, doesn't
+// contend with live-view/recording). See wiki/entities/lpr-camera.md.
+const streamField: ConfigField = {
+  key: "stream",
+  label: "Snapshot stream",
+  type: "select",
+  required: false,
+  default: "1",
+  options: [
+    { value: "1", label: "Main (01)" },
+    { value: "2", label: "Sub (02)" },
+  ],
+};
+
+// Dahua has no stream selector (its CGI snapshot isn't stream-encoded in the path).
+const cameraConfigFields = [
+  hostField,
+  portField(80),
+  usernameField,
+  passwordField,
+  channelField,
+];
+const hikvisionConfigFields = [...cameraConfigFields, streamField];
 
 // Hikvision "Alarm Server" PUSH config. The newer firmware (Event → Smart/VCA →
 // "Detection Target: Human/Vehicle", Notify Surveillance Center, Alarm Settings →
@@ -138,15 +227,23 @@ export const hikvisionDriver: CameraDriver = {
   id: "hikvision",
   category: "camera",
   label: "Hikvision camera",
-  description: "Hikvision snapshot via ISAPI (HTTP Digest) + optional Alarm Server event push.",
+  description:
+    "Hikvision snapshot via ISAPI (HTTP Digest) + optional Alarm Server event push.",
   transports: ["tcp-ip"],
   // The camera PULLS snapshots, but with Alarm Server on it ALSO pushes events to us —
   // so it may need the backend push IP at assign time (like the Dingtian).
   pushesToBackend: true,
-  configFields: [...cameraConfigFields, ...alarmPushFields],
+  configFields: [...hikvisionConfigFields, ...alarmPushFields],
-  // ISAPI channel id: <channel><stream>, e.g. ch1 main = 101, ch2 main = 201.
+  // ISAPI channel id: <channel><stream>, e.g. ch1 main = 101, ch1 sub = 102, ch2 main = 201.
-  create: (c) =>
+  // stream 1 → "01" (main), 2 → "02" (sub).
-    new HttpCamera("hikvision", c, (ch) => `/ISAPI/Streaming/channels/${ch}01/picture`),
+  create: (c) => {
+    console.log(c);
+    return new HttpCamera(
+      "hikvision",
+      c,
+      (ch, stream) => `/ISAPI/Streaming/channels/${ch}0${stream}/picture`,
+    );
+  },
 };
 
 export const dahuaDriver: CameraDriver = {
@@ -156,7 +253,12 @@ export const dahuaDriver: CameraDriver = {
   description: "Dahua snapshot via CGI (HTTP Digest).",
   transports: ["tcp-ip"],
   configFields: cameraConfigFields,
-  // Dahua channels are 0-based on the CGI; the admin enters 1-based.
+  // Dahua channels are 0-based on the CGI; the admin enters 1-based. No stream in the
+  // path (the second arg is ignored — Dahua has no main/sub snapshot distinction here).
   create: (c) =>
-    new HttpCamera("dahua", c, (ch) => `/cgi-bin/snapshot.cgi?channel=${Math.max(0, ch - 1)}`),
+    new HttpCamera(
+      "dahua",
+      c,
+      (ch) => `/cgi-bin/snapshot.cgi?channel=${Math.max(0, ch - 1)}`,
+    ),
 };