fix(button-light): back off failed setAux sends — kill the ENETUNREACH hot loop

An unreachable controller rejects the UDP send instantly, and #pump's
failure re-pump retried inline: a tight loop logging hundreds of identical
errors per minute (park-buzi, 2026-07-07). Failed sends now arm a 1s→30s
exponential retry (reset on success); desiredOn keeps tracking the truth
table meanwhile and the armed retry converges to it. Logging is
rate-limited: first failure of a streak in full, then one summary/minute,
one info line on recovery. #finalOff waives the backoff so the last-gasp
OFF on drop/shutdown still gets an immediate try.

Claude-Session: https://claude.ai/code/session_01Xcm6ikLgGoCxxHrxtjkk5V
This commit is contained in:
2026-07-10 08:29:33 +02:00
parent 5287be5278
commit e2d5105da2
2 changed files with 146 additions and 15 deletions
+68 -1
View File
@@ -192,11 +192,78 @@ describe("ButtonLightController truth table", () => {
// First write (initial off) throws — must be swallowed.
expect(() => ctl.start()).not.toThrow();
await flush();
// Subsequent writes work; driving to solid still converges to ON.
// The failure arms a backoff (1s) rather than retrying inline; desired-state
// changes during the window just update the target the retry will assert.
lane(true);
radar(true);
await flush();
expect(ctl.confirmedOf(CONTROLLER)).toBeNull(); // still backing off
await vi.advanceTimersByTimeAsync(1000); // retry fires; aux is healthy again
expect(ctl.confirmedOf(CONTROLLER)).toBe(true); // converged to solid ON
ctl.stop();
});
it("an unreachable controller backs off (1s→30s), not a hot retry loop", async () => {
let attempts = 0;
const aux: AuxOutputDevice = {
async setAux() {
attempts += 1;
throw new Error("send ENETUNREACH 10.0.10.5:60000");
},
};
const errors: string[] = [];
const logger = silentLogger();
(logger as { error: (msg: string) => void }).error = (msg) => errors.push(msg);
const ctl = new ButtonLightController(db, logger, () => aux);
ctl.start(); // initial OFF write → attempt 1 fails at t=0
await flush();
expect(attempts).toBe(1); // the old code hot-looped here
// Failures at t≈0,1,3,7,15,31 (doubling, capped 30s) → 6 attempts in the first
// minute instead of thousands.
await vi.advanceTimersByTimeAsync(60_000);
expect(attempts).toBeGreaterThanOrEqual(5);
expect(attempts).toBeLessThanOrEqual(7);
// Only the FIRST failure was logged so far; the next log is a ≥60s summary.
expect(errors).toHaveLength(1);
await vi.advanceTimersByTimeAsync(35_000); // t≈95s → the t=61s attempt logged a summary
expect(errors.length).toBe(2);
expect(errors[1]).toContain("still failing");
ctl.stop();
});
it("logs a single recovery line and resets the backoff after success", async () => {
let failing = true;
let attempts = 0;
const aux: AuxOutputDevice = {
async setAux() {
attempts += 1;
if (failing) throw new Error("send ENETUNREACH 10.0.10.5:60000");
},
};
const infos: string[] = [];
const logger = silentLogger();
(logger as { info: (msg: string) => void }).info = (msg) => infos.push(msg);
const ctl = new ButtonLightController(db, logger, () => aux);
ctl.start();
await flush();
await vi.advanceTimersByTimeAsync(3_000); // attempts at t=0,1,3 all fail
const failed = attempts;
expect(failed).toBeGreaterThanOrEqual(3);
failing = false; // controller reachable again
await vi.advanceTimersByTimeAsync(8_000); // next armed retry succeeds
expect(ctl.confirmedOf(CONTROLLER)).toBe(false); // OFF asserted on the device
expect(infos.filter((m) => m.includes("recovered"))).toHaveLength(1);
// Backoff reset: a fresh state change sends immediately (no lingering retryAt).
const before = attempts;
lane(true);
radar(true);
await flush();
expect(ctl.confirmedOf(CONTROLLER)).toBe(true);
expect(attempts).toBe(before + 1);
ctl.stop();
});