Files
parking_solution/apps/server/src/button-light.ts
T
julian e2d5105da2 fix(button-light): back off failed setAux sends — kill the ENETUNREACH hot loop
An unreachable controller rejects the UDP send instantly, and #pump's
failure re-pump retried inline: a tight loop logging hundreds of identical
errors per minute (park-buzi, 2026-07-07). Failed sends now arm a 1s→30s
exponential retry (reset on success); desiredOn keeps tracking the truth
table meanwhile and the armed retry converges to it. Logging is
rate-limited: first failure of a streak in full, then one summary/minute,
one info line on recovery. #finalOff waives the backoff so the last-gasp
OFF on drop/shutdown still gets an immediate try.

Claude-Session: https://claude.ai/code/session_01Xcm6ikLgGoCxxHrxtjkk5V
2026-07-10 08:29:33 +02:00

382 lines
17 KiB
TypeScript

import { eq, devices, type Db, type DeviceRow } from "@parking/db";
import type { FastifyBaseLogger } from "fastify";
import { hasAuxOutput, registry, type AuxOutputDevice } from "@parking/devices";
import { deviceEvents, type DeviceInputEvent, type LaneStatusEvent } from "./device-events.js";
import { alertRelaysOf, relayForPresence, type RelaySpec } from "./device-resolve.js";
// Alert (radarAlert) relays — non-barrier indicator lamps, e.g. the entry button's 12 V
// light. Each lamp is a `relays[]` row with event `radarAlert`, driven by ITS trigger
// input vs. the camera "car in zone" signal (the advisory lane-status). A disagreement
// indicator:
// trigger active + lane busy (camera confirms a car) → SOLID on
// trigger active + lane free (radar sees something, no car) → BLINK (~1 Hz)
// otherwise → OFF
// The lamp is a NON-barrier aux output (setAux latch), so holding/blinking it is fine
// — barrier-not-a-door applies only to barriers, which still only pulseOpen. The lamp
// FAILS OFF: any error / shutdown leaves it off, so a dead lamp is "no hint", never a
// misleading solid "go". A controller may have several alert relays (each its own row +
// trigger input), keyed independently. See wiki/concepts/button-light-indicator.md.
type LightState = "off" | "solid" | "blink";
const DEFAULT_BLINK_MS = 500;
// Failed-send retry backoff: 1s doubling to 30s, reset on success. Without this an
// unreachable controller (ENETUNREACH) became a hot loop — the failure re-pump retried
// instantly, thousands of sends + error lines per minute (field incident 2026-07-07).
const RETRY_BASE_MS = 1_000;
const RETRY_MAX_MS = 30_000;
/** After the first failure of a streak, log at most one summary line per this window. */
const FAIL_LOG_EVERY_MS = 60_000;
/** Per-lamp live state for the alert rule (one per radarAlert relay). */
interface LampState {
/** The controller this lamp lives on (its deviceId) — for resolving the aux adapter. */
readonly controllerId: string;
/** Alert relay row (relay #, triggerInput, blink ms). Mutable: #reconcile updates it in
* place when the admin changes the alert config without a restart. */
spec: RelaySpec;
/** Is the lamp's trigger input (the radar) currently active? */
present: boolean;
/** The high-level state we're rendering (to avoid restarting a running blink). */
rendered: LightState | null;
/** Active blink timer, if blinking. */
blink: ReturnType<typeof setInterval> | null;
/** Blink phase (true = currently on). */
blinkOn: boolean;
/** The output we WANT the relay to be in. The serialized worker drives the device
* toward this. The blink timer only flips this flag — it never sends directly. */
desiredOn: boolean;
/** The output we last CONFIRMED on the device (after a successful send). null = unknown. */
confirmedOn: boolean | null;
/** True while a send is in flight for this lamp — serializes UDP so on/off can't
* overlap or reorder (UDP is unordered; concurrent toggles left the relay stuck). */
sending: boolean;
/** Consecutive failed sends (0 = healthy). Drives the backoff delay + log summaries. */
failCount: number;
/** Epoch ms before which #pump must not send (0 = no backoff). The armed retry
* timer re-pumps when it elapses; desired-state changes in between just update
* `desiredOn` and are picked up by that same retry. */
retryAt: number;
/** The armed backoff retry, if any. */
retryTimer: ReturnType<typeof setTimeout> | null;
/** Epoch ms of the last failure line we actually logged (rate-limits the flood). */
lastFailLogAt: number;
}
/** Resolves a controller's live aux-output adapter. The default goes through the
* driver registry; tests inject a spy. Returns null when the controller has no
* aux-output capability (or won't build). */
export type AuxResolver = (controllerId: string) => AuxOutputDevice | null;
export class ButtonLightController {
readonly #db: Db;
readonly #logger: FastifyBaseLogger;
readonly #resolveAux: AuxResolver;
/** Per-lamp state, keyed by `${controllerId}:${relay}` (a controller may have several). */
readonly #lamps = new Map<string, LampState>();
/** Latest lane status — a camera-confirmed car in the entry / exit zone. A lamp locks
* SOLID off its OWN lane's camera (`spec.lockLane`), so an exit radar's lamp tracks the
* exit camera, not the entry one. */
#entryBusy = false;
#exitBusy = false;
/** Controllers we've already warned lack the aux-output capability (warn once). */
readonly #warned = new Set<string>();
#unsubInput: (() => void) | null = null;
#unsubLane: (() => void) | null = null;
constructor(db: Db, logger: FastifyBaseLogger, resolveAux?: AuxResolver) {
this.#db = db;
this.#logger = logger;
this.#resolveAux = resolveAux ?? ((id) => this.#auxFromRegistry(id));
}
/** Subscribe to radar input edges + lane status, and initialise every lamp OFF. */
start(): void {
this.#reconcile();
// All lamps start OFF (known-safe baseline) regardless of prior device state.
for (const lamp of this.#lamps.values()) this.#apply(lamp);
this.#unsubInput = deviceEvents.onInput((e) => this.#onInput(e));
this.#unsubLane = deviceEvents.onLaneStatus((s) => this.#onLane(s));
}
/** Reconcile the lamp map with the CURRENT device config (the booth can add/change a
* button light without a server restart). Mirrors DeviceMonitor, which re-reads the
* device set each tick. Adds lamps for newly-configured controllers, updates the spec
* (relay #, blink ms) in place — preserving live `present`/blink state — and drops
* lamps whose controller lost its buttonLight or was disabled. Called at start() and
* before handling each event, so a just-saved lamp takes effect immediately. */
#reconcile(): void {
const rows = this.#db.select().from(devices).where(eq(devices.category, "access")).all();
const seen = new Set<string>();
for (const row of rows) {
if (!row.enabled) continue;
for (const spec of alertRelaysOf(row)) {
const key = lampKey(row.id, spec.relay);
seen.add(key);
const existing = this.#lamps.get(key);
if (existing) {
existing.spec = spec; // pick up a changed trigger input / blink cadence
} else {
this.#lamps.set(key, {
controllerId: row.id,
spec,
present: false,
rendered: null,
blink: null,
blinkOn: false,
desiredOn: false,
confirmedOn: null,
sending: false,
failCount: 0,
retryAt: 0,
retryTimer: null,
lastFailLogAt: 0,
});
}
}
}
// Drop lamps whose controller no longer declares one (or was disabled/removed).
for (const [key, lamp] of this.#lamps) {
if (seen.has(key)) continue;
this.#disarm(lamp);
this.#finalOff(lamp); // best-effort fail-OFF before forgetting it
this.#lamps.delete(key);
}
}
/** A radar (presence) edge updates that controller's `present` flag. We resolve the
* edge the SAME way the entry flow does (relayForPresence on an entry/both relay),
* so the lamp and the one-car-one-ticket gate always agree on "a car is here". */
#onInput(e: DeviceInputEvent): void {
// Reconcile first so a lamp added/changed since boot (no restart) is picked up.
this.#reconcile();
const present = e.edge === "on";
for (const lamp of this.#lamps.values()) {
if (lamp.controllerId !== e.deviceId) continue;
// A lamp's trigger is its own `triggerInput`; if unset, fall back to the controller's
// entry-relay presence terminal (resolved the SAME way the entry flow does) so the
// lamp and the one-car-one-ticket gate always agree on "a car is here".
const trigger =
lamp.spec.triggerInput ?? relayForPresence(this.#db, e.deviceId, e.input)?.presenceInput;
if (trigger !== e.input) continue; // not this lamp's trigger terminal
if (present === lamp.present) continue;
lamp.present = present;
this.#apply(lamp);
}
}
/** Lane status changed: a camera-confirmed car in the entry and/or exit zone. */
#onLane(s: LaneStatusEvent): void {
if (s.entry === this.#entryBusy && s.exit === this.#exitBusy) return;
this.#entryBusy = s.entry;
this.#exitBusy = s.exit;
// Re-render every lamp (each picks its own lane's camera in #apply).
for (const lamp of this.#lamps.values()) this.#apply(lamp);
}
/** Compute + render the target state for one lamp. Drives are fire-and-forget (the
* timer/state machine is synchronous; the UDP write resolves on its own). */
#apply(lamp: LampState): void {
// SOLID only once THIS lamp's lane camera confirms a car (default entry).
const laneBusy = lamp.spec.lockLane === "exit" ? this.#exitBusy : this.#entryBusy;
const target: LightState = !lamp.present ? "off" : laneBusy ? "solid" : "blink";
if (target === lamp.rendered) return; // already rendering this state
// Tear down any running blink before switching states.
if (lamp.blink) {
clearInterval(lamp.blink);
lamp.blink = null;
}
lamp.rendered = target;
if (target === "off") {
lamp.desiredOn = false;
this.#pump(lamp);
} else if (target === "solid") {
lamp.desiredOn = true;
this.#pump(lamp);
} else {
// BLINK: a wall-clock timer flips ONLY the desired flag; #pump does the actual
// (serialized) UDP send. A symmetric cadence uses one interval; an asymmetric one
// re-arms each phase with its own duration. Sends never overlap or reorder, so the
// relay can't get stuck on a stale packet.
const onMs = lamp.spec.blinkOnMs && lamp.spec.blinkOnMs > 0 ? lamp.spec.blinkOnMs : DEFAULT_BLINK_MS;
const offMs = lamp.spec.blinkOffMs && lamp.spec.blinkOffMs > 0 ? lamp.spec.blinkOffMs : DEFAULT_BLINK_MS;
lamp.blinkOn = true;
lamp.desiredOn = true;
const tick = () => {
lamp.blinkOn = !lamp.blinkOn;
lamp.desiredOn = lamp.blinkOn;
this.#pump(lamp);
if (onMs !== offMs && lamp.blink) {
clearInterval(lamp.blink);
lamp.blink = setInterval(tick, lamp.blinkOn ? onMs : offMs);
lamp.blink.unref?.();
}
};
lamp.blink = setInterval(tick, onMs);
lamp.blink.unref?.();
this.#pump(lamp);
}
}
/** Serialized per-lamp worker: drive the relay toward `desiredOn`, one UDP send at a
* time. Because UDP is unordered, concurrent on/off sends previously raced and left
* the relay stuck on a stale packet. Here a single in-flight send is guaranteed
* (`sending` guard); when it resolves, if the desired state moved on we send again —
* so the LAST desired state is always the one finally asserted on the device.
*
* Failures back off (1s → 30s, reset on success) instead of retrying inline: an
* unreachable controller rejects instantly, and an immediate re-pump was a hot loop.
* During backoff `desiredOn` keeps tracking the truth table; the armed retry timer
* converges to whatever it says when it fires. Only the FIRST failure of a streak is
* logged, then one summary per minute, and an info line on recovery. */
#pump(lamp: LampState): void {
if (lamp.sending) return; // a send is already in flight; it'll re-check on completion
if (lamp.confirmedOn === lamp.desiredOn) return; // already there — no redundant UDP
if (Date.now() < lamp.retryAt) return; // backing off — the retry timer will re-pump
const aux = this.#resolveAux(lamp.controllerId);
if (!aux) return;
const target = lamp.desiredOn;
lamp.sending = true;
void aux
.setAux(lamp.spec.relay, target)
.then(() => {
lamp.confirmedOn = target;
if (lamp.failCount > 0) {
this.#logger.info(
`button-light setAux recovered (${lamp.controllerId} R${lamp.spec.relay}) after ${lamp.failCount} failed attempts`,
);
}
lamp.failCount = 0;
lamp.retryAt = 0;
lamp.lastFailLogAt = 0;
})
.catch((err: unknown) => {
// Leave confirmedOn unchanged so the armed retry re-asserts the (then-current)
// desired state. Never escalates — a dead lamp is "no hint", never a fault.
lamp.failCount += 1;
const delay = Math.min(RETRY_BASE_MS * 2 ** (lamp.failCount - 1), RETRY_MAX_MS);
lamp.retryAt = Date.now() + delay;
const now = Date.now();
if (lamp.failCount === 1 || now - lamp.lastFailLogAt >= FAIL_LOG_EVERY_MS) {
lamp.lastFailLogAt = now;
const streak =
lamp.failCount > 1 ? ` — still failing (attempt ${lamp.failCount}, retrying ≤${RETRY_MAX_MS / 1000}s)` : "";
this.#logger.error(
`button-light setAux failed (${lamp.controllerId} R${lamp.spec.relay}): ${(err as Error).message}${streak}`,
);
}
if (lamp.retryTimer) clearTimeout(lamp.retryTimer);
lamp.retryTimer = setTimeout(() => {
lamp.retryTimer = null;
this.#pump(lamp);
}, delay);
lamp.retryTimer.unref?.();
})
.finally(() => {
lamp.sending = false;
// Desired state may have changed while we were busy — re-pump to converge (the
// backoff gate above makes this a no-op right after a failure). This is what
// makes the final state authoritative.
if (lamp.confirmedOn !== lamp.desiredOn) this.#pump(lamp);
});
}
/** Build the live aux-output adapter for a controller, or null (logged once). */
#auxFromRegistry(controllerId: string): AuxOutputDevice | null {
const row = this.#db.select().from(devices).where(eq(devices.id, controllerId)).get();
if (!row) return null;
const driver = registry.get(row.driverId);
if (!driver) return null;
let device: unknown;
try {
device = driver.create(row.config as never);
} catch {
return null;
}
if (!hasAuxOutput(device)) {
if (!this.#warned.has(controllerId)) {
this.#warned.add(controllerId);
this.#logger.warn(`button-light: controller ${controllerId} (${row.driverId}) has no aux-output — lamp ignored`);
}
return null;
}
return device;
}
/** Unsubscribe, stop all blink timers, and best-effort drive every lamp OFF. */
stop(): void {
this.#unsubInput?.();
this.#unsubLane?.();
this.#unsubInput = null;
this.#unsubLane = null;
for (const lamp of this.#lamps.values()) {
this.#disarm(lamp);
// Best-effort fail-OFF on shutdown.
this.#finalOff(lamp);
}
}
/** Stop a lamp's timers (blink + backoff retry) without touching the device. */
#disarm(lamp: LampState): void {
if (lamp.blink) {
clearInterval(lamp.blink);
lamp.blink = null;
}
if (lamp.retryTimer) {
clearTimeout(lamp.retryTimer);
lamp.retryTimer = null;
}
}
/** Drive a lamp OFF as a one-shot (used when dropping/stopping a lamp): set desired
* OFF and pump. The serialized worker still applies, so this can't collide with an
* in-flight send — it converges to OFF. Any backoff is waived so the last-gasp OFF
* gets one immediate try (a lamp mid-backoff may just have recovered). */
#finalOff(lamp: LampState): void {
lamp.desiredOn = false;
lamp.retryAt = 0;
this.#pump(lamp);
}
/** Test seam: current high-level state being rendered for a lamp (controller + relay).
* `relay` defaults to the controller's only/first alert relay for single-lamp tests. */
stateOf(controllerId: string, relay?: number): LightState | null {
return this.#lamp(controllerId, relay)?.rendered ?? null;
}
/** Test seam: the state last CONFIRMED on the device for a lamp (after a successful
* send). null = unknown / nothing sent yet. `relay` defaults to the only alert relay. */
confirmedOf(controllerId: string, relay?: number): boolean | null {
return this.#lamp(controllerId, relay)?.confirmedOn ?? null;
}
/** Resolve a lamp by controller + relay. When `relay` is omitted, returns the
* controller's single lamp (the common single-alert case); ambiguous if several. */
#lamp(controllerId: string, relay?: number): LampState | undefined {
if (relay != null) return this.#lamps.get(lampKey(controllerId, relay));
for (const lamp of this.#lamps.values()) if (lamp.controllerId === controllerId) return lamp;
return undefined;
}
}
/** Composite key for the lamp map (a controller may carry several alert relays). */
function lampKey(controllerId: string, relay: number): string {
return `${controllerId}:${relay}`;
}
/** Build a controller row's live aux device (exported for reuse/tests). */
export function buildAux(db: Db, row: DeviceRow): AuxOutputDevice | null {
const driver = registry.get(row.driverId);
if (!driver) return null;
try {
const device = driver.create(row.config as never);
return hasAuxOutput(device) ? device : null;
} catch {
return null;
}
}