From f0fd15bb88adb49d49b4fdb81fc9e59afca26623 Mon Sep 17 00:00:00 2001 From: Julian Cuni Date: Fri, 26 Jun 2026 16:46:46 +0200 Subject: [PATCH] fix(camera): selectable snapshot stream + retry transient 503 Device Busy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Hikvision DS-2CD1047G3H-LIU returned HTTP 503 (statusCode 2 / deviceBusy) on EVERY main-stream snapshot — its main encoder is persistently saturated. Probed on hardware: channels/101/picture → 503 on 5 consecutive tries, while channels/102/picture (sub stream) → 200 clean JPEG every time. A retry loop can't fix a persistent busy; the real fix is stream selection. - Add a `stream` config field to the Hikvision driver (1=main, default for back-compat; 2=sub). ISAPI channel id is (101 main, 102 sub). Verified live: setting the G3H to Sub flips its status degraded→ready (14.7KB JPEG in ~87ms). - captureSnapshot also retries the TRANSIENT case (503/500, linear backoff 250/500/750ms ×4) then fails naming it "(device busy)"; does NOT retry 401/404 (config errors won't self-heal). Complements captureSnapshotShared (concurrent de-dup). healthCheck still reports a live 503 as degraded (surfaces a saturated main stream rather than hiding it). Tests: camera.test.ts (10) — retry behaviour + main/sub path selection. Claude-Session: https://claude.ai/code/session_01Xcm6ikLgGoCxxHrxtjkk5V --- packages/devices/src/drivers/camera.test.ts | 115 ++++++++++++++++ packages/devices/src/drivers/camera.ts | 140 +++++++++++++++++--- 2 files changed, 236 insertions(+), 19 deletions(-) create mode 100644 packages/devices/src/drivers/camera.test.ts diff --git a/packages/devices/src/drivers/camera.test.ts b/packages/devices/src/drivers/camera.test.ts new file mode 100644 index 0000000..7899c47 --- /dev/null +++ b/packages/devices/src/drivers/camera.test.ts @@ -0,0 +1,115 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { DigestGetResult } from "./http-digest.js"; + +// The HTTP layer is mocked so the camera driver's RETRY logic is tested without a +// network. Hikvision returns 503 "Device Busy" (sometimes 500) transiently when its +// snapshot encoder is occupied — captureSnapshot must retry those and succeed, but +// fail FAST on a config error (401 auth / 404 path). See camera.ts. + +const digestGet = vi.fn<(...a: unknown[]) => Promise>(); +vi.mock("./http-digest.js", () => ({ digestGet: (...a: unknown[]) => digestGet(...a) })); + +// Import the driver AFTER the mock is registered. +const { hikvisionDriver } = await import("./camera.js"); + +function reply(status: number, body = "jpeg-bytes"): DigestGetResult { + return { status, contentType: "image/jpeg", body: Buffer.from(body) }; +} + +function makeCamera() { + return hikvisionDriver.create({ host: "10.0.10.12", port: 80, username: "admin", password: "x", channel: 1 }); +} + +beforeEach(() => { + digestGet.mockReset(); + vi.useFakeTimers(); +}); +afterEach(() => { + vi.useRealTimers(); +}); + +describe("hikvision captureSnapshot — 503 Device Busy retry", () => { + it("retries a transient 503 and succeeds", async () => { + digestGet + .mockResolvedValueOnce(reply(503)) + .mockResolvedValueOnce(reply(503)) + .mockResolvedValueOnce(reply(200, "the-frame")); + const cam = makeCamera(); + const p = cam.captureSnapshot({ direction: "entry" }); + await vi.runAllTimersAsync(); // let the backoff sleeps fire + const shot = await p; + expect(shot.bytes.toString()).toBe("the-frame"); + expect(digestGet).toHaveBeenCalledTimes(3); // 503, 503, 200 + }); + + it("also retries a transient 500", async () => { + digestGet.mockResolvedValueOnce(reply(500)).mockResolvedValueOnce(reply(200)); + const cam = makeCamera(); + const p = cam.captureSnapshot({ direction: "entry" }); + await vi.runAllTimersAsync(); + await p; + expect(digestGet).toHaveBeenCalledTimes(2); + }); + + it("gives up after the attempt cap, naming it 'device busy'", async () => { + digestGet.mockResolvedValue(reply(503)); // always busy + const cam = makeCamera(); + // Attach the rejection assertion BEFORE flushing timers so the rejection always + // has a handler (no unhandled-rejection noise), then drive the backoff sleeps. + const assertion = expect(cam.captureSnapshot({ direction: "entry" })).rejects.toThrow(/HTTP 503 \(device busy\)/); + await vi.runAllTimersAsync(); + await assertion; + expect(digestGet).toHaveBeenCalledTimes(4); // SNAPSHOT_MAX_ATTEMPTS + }); + + it("does NOT retry a 401 (auth error self-won't-heal) — fails fast", async () => { + digestGet.mockResolvedValue(reply(401)); + const cam = makeCamera(); + const assertion = expect(cam.captureSnapshot({ direction: "entry" })).rejects.toThrow(/HTTP 401/); + await vi.runAllTimersAsync(); + await assertion; + expect(digestGet).toHaveBeenCalledTimes(1); // no retry + }); + + it("does NOT retry a 404 (wrong path/channel) — fails fast", async () => { + digestGet.mockResolvedValue(reply(404)); + const cam = makeCamera(); + const assertion = expect(cam.captureSnapshot({ direction: "entry" })).rejects.toThrow(/HTTP 404/); + await vi.runAllTimersAsync(); + await assertion; + expect(digestGet).toHaveBeenCalledTimes(1); + }); + + it("succeeds first try with no retry on a clean 200", async () => { + digestGet.mockResolvedValue(reply(200)); + const cam = makeCamera(); + const shot = await cam.captureSnapshot({ direction: "entry" }); + expect(shot.contentType).toBe("image/jpeg"); + expect(digestGet).toHaveBeenCalledTimes(1); + }); +}); + +describe("hikvision snapshot stream selection (main vs sub)", () => { + function pathFor(config: Record): string { + digestGet.mockReset(); + digestGet.mockResolvedValue(reply(200)); + hikvisionDriver.create(config as never).captureSnapshot({ direction: "entry" }); + return String((digestGet.mock.calls[0]![0] as { path: string }).path); + } + + it("defaults to the MAIN stream (…/channels/101/picture) — back-compat", () => { + expect(pathFor({ host: "1.2.3.4", channel: 1 })).toBe("/ISAPI/Streaming/channels/101/picture"); + }); + + it("stream=2 selects the SUB stream (…/channels/102/picture) — the G3H 503 fix", () => { + expect(pathFor({ host: "1.2.3.4", channel: 1, stream: 2 })).toBe("/ISAPI/Streaming/channels/102/picture"); + }); + + it("honours the channel number with the stream (ch2 sub = 202)", () => { + expect(pathFor({ host: "1.2.3.4", channel: 2, stream: 2 })).toBe("/ISAPI/Streaming/channels/202/picture"); + }); + + it("an invalid stream falls back to main (1)", () => { + expect(pathFor({ host: "1.2.3.4", channel: 1, stream: 9 })).toBe("/ISAPI/Streaming/channels/101/picture"); + }); +}); diff --git a/packages/devices/src/drivers/camera.ts b/packages/devices/src/drivers/camera.ts index 6370a71..9277960 100644 --- a/packages/devices/src/drivers/camera.ts +++ b/packages/devices/src/drivers/camera.ts @@ -1,6 +1,17 @@ -import type { CameraDevice, DeviceHealth, Snapshot, SnapshotContext } from "../interfaces.js"; +import type { + CameraDevice, + DeviceHealth, + Snapshot, + SnapshotContext, +} from "../interfaces.js"; import type { CameraDriver, ConfigField, DeviceConfig } from "../registry.js"; -import { hostField, passwordField, portField, usernameField, stubLog } from "./common.js"; +import { + hostField, + passwordField, + portField, + usernameField, + stubLog, +} from "./common.js"; import { digestGet } from "./http-digest.js"; // Camera drivers — entry/exit snapshot-on-event. The host pulls a still over @@ -15,12 +26,32 @@ import { digestGet } from "./http-digest.js"; const DEFAULT_TIMEOUT_MS = 8000; +// Hikvision returns HTTP 503 (statusCode 2 / "Device Busy" / subStatus deviceBusy) — +// and occasionally 500 — when its snapshot encoder is momentarily occupied (another +// snapshot in flight, a stream starting, on-camera VCA). It is TRANSIENT: a retry a +// few hundred ms later succeeds. The newer G3H sensors (e.g. DS-2CD1047G3H) hit it +// more readily. So a standalone capture retries a few times before giving up; we do +// NOT retry config errors (401 auth, 404 path/channel) — those won't self-heal. +// (Concurrent same-camera hits are separately de-duped by captureSnapshotShared in +// the server.) See wiki/entities/lpr-camera.md ("503 Device Busy"). +const SNAPSHOT_RETRY_STATUSES = new Set([500, 503]); +const SNAPSHOT_MAX_ATTEMPTS = 4; +const SNAPSHOT_RETRY_BASE_MS = 250; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + class HttpCamera implements CameraDevice { readonly #host: string; readonly #port: number; readonly #user: string; readonly #password: string; readonly #channel: number; + /** Hikvision stream within the channel: 1 = main (high-res), 2 = sub (lighter). + * Some models (e.g. the G3H) keep the MAIN encoder saturated and return a + * persistent 503 deviceBusy on the main-stream snapshot, while the sub-stream + * serves fine — so this is selectable. Ignored by drivers (Dahua) that don't + * encode a stream in the path. See wiki/entities/lpr-camera.md ("503 Device Busy"). */ + readonly #stream: number; readonly #timeout: number; // Source outbound from the device-facing NIC on a multi-homed host (the // multi-subnet source-address trap — see wiki/concepts/wsl-dev-networking.md). @@ -29,16 +60,20 @@ class HttpCamera implements CameraDevice { constructor( readonly driverId: string, config: DeviceConfig, - /** Builds the snapshot path from the configured channel. */ - private readonly snapshotPath: (channel: number) => string, + /** Builds the snapshot path from the configured channel + stream (1=main, 2=sub). */ + private readonly snapshotPath: (channel: number, stream: number) => string, ) { this.#host = String(config.host); this.#port = Number(config.port ?? 80); this.#user = String(config.username ?? ""); this.#password = String(config.password ?? ""); this.#channel = Number(config.channel ?? 1); + // 1 = main, 2 = sub. Clamp to those two; default main for back-compat. + this.#stream = Number(config.stream) === 2 ? 2 : 1; this.#timeout = Number(config.timeoutMs ?? DEFAULT_TIMEOUT_MS); - this.#localAddress = config.localAddress ? String(config.localAddress) : undefined; + this.#localAddress = config.localAddress + ? String(config.localAddress) + : undefined; } async connect(): Promise {} @@ -49,8 +84,13 @@ class HttpCamera implements CameraDevice { // frame: it exercises reachability + auth + the path/channel in one shot. try { const res = await this.#get(); - if (res.status === 200) return { status: "ready", detail: `${res.body.length} bytes` }; - if (res.status === 401) return { status: "degraded", detail: "auth rejected (check username/password)" }; + if (res.status === 200) + return { status: "ready", detail: `${res.body.length} bytes` }; + if (res.status === 401) + return { + status: "degraded", + detail: "auth rejected (check username/password)", + }; return { status: "degraded", detail: `HTTP ${res.status}` }; } catch (err) { return { status: "offline", detail: (err as Error).message }; @@ -58,13 +98,37 @@ class HttpCamera implements CameraDevice { } async captureSnapshot(ctx: SnapshotContext): Promise { - const res = await this.#get(); + // Retry transient "Device Busy" (503/500); a config error (401/404) fails fast. + let res = await this.#get(); + for ( + let attempt = 1; + res.status !== 200 && + SNAPSHOT_RETRY_STATUSES.has(res.status) && + attempt < SNAPSHOT_MAX_ATTEMPTS; + attempt++ + ) { + // Linear backoff (250/500/750ms) — the encoder frees within a frame or two. + await sleep(SNAPSHOT_RETRY_BASE_MS * attempt); + stubLog( + this.driverId, + `captureSnapshot ${ctx.direction} retry ${attempt} (was HTTP ${res.status})`, + ); + res = await this.#get(); + } if (res.status !== 200) { + // Name the busy case so the operator/telemetry can tell "camera busy" from a + // real fault (offline / auth / wrong path). + const busy = SNAPSHOT_RETRY_STATUSES.has(res.status) + ? " (device busy)" + : ""; throw new Error( - `${this.driverId} snapshot failed (${ctx.direction}): HTTP ${res.status}`, + `${this.driverId} snapshot failed (${ctx.direction}): HTTP ${res.status}${busy}`, ); } - stubLog(this.driverId, `captureSnapshot ${ctx.direction} (${res.body.length} bytes)`); + stubLog( + this.driverId, + `captureSnapshot ${ctx.direction} (${res.body.length} bytes)`, + ); return { bytes: res.body, contentType: res.contentType || "image/jpeg", @@ -76,7 +140,7 @@ class HttpCamera implements CameraDevice { return digestGet({ host: this.#host, port: this.#port, - path: this.snapshotPath(this.#channel), + path: this.snapshotPath(this.#channel, this.#stream), user: this.#user, password: this.#password, timeoutMs: this.#timeout, @@ -93,7 +157,32 @@ const channelField: ConfigField = { default: 1, }; -const cameraConfigFields = [hostField, portField(80), usernameField, passwordField, channelField]; +// Hikvision stream-within-channel for the snapshot: main (01) is full-res; sub (02) +// is lighter. Default MAIN (back-compat). Switch to SUB when the main encoder is +// saturated and returns a persistent 503 deviceBusy (seen on DS-2CD1047G3H-LIU) — +// the sub-stream is also the better fit for snapshot/ANPR (smaller, faster, doesn't +// contend with live-view/recording). See wiki/entities/lpr-camera.md. +const streamField: ConfigField = { + key: "stream", + label: "Snapshot stream", + type: "select", + required: false, + default: "1", + options: [ + { value: "1", label: "Main (01)" }, + { value: "2", label: "Sub (02)" }, + ], +}; + +// Dahua has no stream selector (its CGI snapshot isn't stream-encoded in the path). +const cameraConfigFields = [ + hostField, + portField(80), + usernameField, + passwordField, + channelField, +]; +const hikvisionConfigFields = [...cameraConfigFields, streamField]; // Hikvision "Alarm Server" PUSH config. The newer firmware (Event → Smart/VCA → // "Detection Target: Human/Vehicle", Notify Surveillance Center, Alarm Settings → @@ -138,15 +227,23 @@ export const hikvisionDriver: CameraDriver = { id: "hikvision", category: "camera", label: "Hikvision camera", - description: "Hikvision snapshot via ISAPI (HTTP Digest) + optional Alarm Server event push.", + description: + "Hikvision snapshot via ISAPI (HTTP Digest) + optional Alarm Server event push.", transports: ["tcp-ip"], // The camera PULLS snapshots, but with Alarm Server on it ALSO pushes events to us — // so it may need the backend push IP at assign time (like the Dingtian). pushesToBackend: true, - configFields: [...cameraConfigFields, ...alarmPushFields], - // ISAPI channel id: , e.g. ch1 main = 101, ch2 main = 201. - create: (c) => - new HttpCamera("hikvision", c, (ch) => `/ISAPI/Streaming/channels/${ch}01/picture`), + configFields: [...hikvisionConfigFields, ...alarmPushFields], + // ISAPI channel id: , e.g. ch1 main = 101, ch1 sub = 102, ch2 main = 201. + // stream 1 → "01" (main), 2 → "02" (sub). + create: (c) => { + console.log(c); + return new HttpCamera( + "hikvision", + c, + (ch, stream) => `/ISAPI/Streaming/channels/${ch}0${stream}/picture`, + ); + }, }; export const dahuaDriver: CameraDriver = { @@ -156,7 +253,12 @@ export const dahuaDriver: CameraDriver = { description: "Dahua snapshot via CGI (HTTP Digest).", transports: ["tcp-ip"], configFields: cameraConfigFields, - // Dahua channels are 0-based on the CGI; the admin enters 1-based. + // Dahua channels are 0-based on the CGI; the admin enters 1-based. No stream in the + // path (the second arg is ignored — Dahua has no main/sub snapshot distinction here). create: (c) => - new HttpCamera("dahua", c, (ch) => `/cgi-bin/snapshot.cgi?channel=${Math.max(0, ch - 1)}`), + new HttpCamera( + "dahua", + c, + (ch) => `/cgi-bin/snapshot.cgi?channel=${Math.max(0, ch - 1)}`, + ), };