fix release v2 recovery flows

packages/request-handler/src/docker.ts

@@ -15,6 +15,31 @@ import { createDockerLogStreamProcessor, decodeDockerLogs } from "./docker-log-d
 import { createWidgetRequestHandler } from "./lib/widget-request-handler";
 
 const logger = createLogger({ module: "dockerRequestHandler" });
+export const dockerWidgetEndpointTimeoutMs = 5_000;
+
+const withDockerTimeoutAsync = async <T>(
+  operation: (signal: AbortSignal) => Promise<T>,
+  message: string,
+  timeoutMs: number,
+) => {
+  const controller = new AbortController();
+  const timeoutError = new Error(message);
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+
+  try {
+    return await Promise.race([
+      operation(controller.signal),
+      new Promise<never>((_, reject) => {
+        timeout = setTimeout(() => {
+          controller.abort(timeoutError);
+          reject(timeoutError);
+        }, timeoutMs);
+      }),
+    ]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+};
 
 const isDemoMode = ["1", "yes", "t", "true"].includes((process.env.DEMO_MODE ?? "").toLowerCase());
 
@@ -275,11 +300,15 @@ export const streamContainerLogsAsync = async (
   };
 };
 
-export async function getContainersWithStatsAsync() {
+export async function getContainersWithStatsAsync(timeoutMs = dockerWidgetEndpointTimeoutMs) {
   const dockerInstances = DockerSingleton.getInstances();
   const results = await Promise.allSettled(
     dockerInstances.map(async ({ instance, host, endpointId, endpointName }) => {
-      const instanceContainers = await instance.listContainers({ all: true });
+      const instanceContainers = await withDockerTimeoutAsync(
+        async (signal) => await instance.listContainers({ all: true, abortSignal: signal }),
+        `Timed out listing containers from Docker host ${host}`,
+        timeoutMs,
+      );
       return instanceContainers
         .filter((container) => !(dockerLabels.hide in container.Labels))
         .map((container) => ({ ...container, instance: host, endpointId, endpointName }));
@@ -329,16 +358,20 @@ export async function getContainersWithStatsAsync() {
     const instance = DockerSingleton.findInstance(container.endpointId)?.instance;
     if (!instance) return null;
 
-    const stats = await instance
-      .getContainer(container.Id)
-      .stats({ stream: false, "one-shot": true })
-      .catch(
-        () =>
-          ({
-            cpu_stats: { online_cpus: 0, cpu_usage: { total_usage: 0 }, system_cpu_usage: 0 },
-            memory_stats: { usage: 0 },
-          }) as ContainerStats,
-      );
+    const stats = await withDockerTimeoutAsync(
+      async (signal) => {
+        const options = { stream: false as const, "one-shot": true, abortSignal: signal };
+        return await instance.getContainer(container.Id).stats(options);
+      },
+      `Timed out reading Docker container stats for ${container.Id}`,
+      timeoutMs,
+    ).catch(
+      () =>
+        ({
+          cpu_stats: { online_cpus: 0, cpu_usage: { total_usage: 0 }, system_cpu_usage: 0 },
+          memory_stats: { usage: 0 },
+        }) as ContainerStats,
+    );
 
     const cpuUsage = calculateCpuUsage(stats);
     const memoryUsage = calculateMemoryUsage(stats);

packages/request-handler/src/test/docker.spec.ts

@@ -2,8 +2,9 @@ import type { ContainerStats } from "dockerode";
 import { describe, expect, test, vi } from "vitest";
 
 import { createDb } from "@homarr/db/test";
+import { DockerSingleton } from "@homarr/docker";
 
-import { calculateCpuUsage, calculateMemoryUsage } from "../docker";
+import { calculateCpuUsage, calculateMemoryUsage, getContainersWithStatsAsync } from "../docker";
 
 vi.mock("@homarr/db", async (importActual) => {
   // eslint-disable-next-line @typescript-eslint/consistent-type-imports
@@ -119,3 +120,76 @@ describe("calculateMemoryUsage", () => {
     expect(calculateMemoryUsage(stats)).toBe(512);
   });
 });
+
+describe("getContainersWithStatsAsync", () => {
+  test("marks a timed-out endpoint unavailable without blocking healthy endpoints", async () => {
+    vi.useFakeTimers();
+    try {
+      vi.spyOn(DockerSingleton, "getInstances").mockReturnValue([
+        createDockerInstance("stalled", () => new Promise(() => undefined)),
+        createDockerInstance("healthy", async () => []),
+      ] as never);
+
+      const resultPromise = getContainersWithStatsAsync(50);
+      await vi.advanceTimersByTimeAsync(50);
+      const result = await resultPromise;
+
+      expect(result.endpoints).toEqual([
+        expect.objectContaining({ id: "stalled", status: "unavailable" }),
+        expect.objectContaining({ id: "healthy", status: "available" }),
+      ]);
+    } finally {
+      vi.useRealTimers();
+      vi.restoreAllMocks();
+    }
+  });
+
+  test("returns a container with zeroed stats when its stats request times out", async () => {
+    vi.useFakeTimers();
+    try {
+      const dockerInstance = createDockerInstance("healthy", async () => [
+        {
+          Id: "container-1",
+          Image: "sonarr:latest",
+          Labels: {},
+          Names: ["/sonarr"],
+          State: "running",
+          Ports: [],
+        },
+      ]);
+      let observedSignal: AbortSignal | undefined;
+      dockerInstance.instance.getContainer = (() => ({
+        stats: (options: { abortSignal?: AbortSignal }) => {
+          observedSignal = options.abortSignal;
+          return new Promise(() => undefined);
+        },
+      })) as never;
+      vi.spyOn(DockerSingleton, "getInstances").mockReturnValue([dockerInstance] as never);
+      vi.spyOn(DockerSingleton, "findInstance").mockReturnValue(dockerInstance as never);
+
+      const resultPromise = getContainersWithStatsAsync(50);
+      await vi.advanceTimersByTimeAsync(50);
+      const result = await resultPromise;
+
+      expect(observedSignal?.aborted).toBe(true);
+      expect(result.containers).toEqual([expect.objectContaining({ id: "container-1", cpuUsage: 0, memoryUsage: 0 })]);
+    } finally {
+      vi.useRealTimers();
+      vi.restoreAllMocks();
+    }
+  });
+});
+
+const createDockerInstance = (endpointId: string, listContainers: () => Promise<unknown[]>) => ({
+  endpointId,
+  endpointName: endpointId,
+  host: endpointId,
+  descriptor: {
+    kind: "docker",
+    transport: { type: "socket" },
+    capabilities: ["inventory"],
+    source: "environment",
+    scope: "admin",
+  },
+  instance: { listContainers, getContainer: () => ({}) },
+});