fix(beszel): add reliable live stats and filesystem UI (#6289)

packages/integrations/src/beszel/beszel-integration.ts

@@ -19,15 +19,66 @@ import type {
   BeszelSystemDetails,
   BeszelSystemdService,
   BeszelSystemStatsRecord,
+  BeszelSystemStats,
+  BeszelContainerStats,
   CreateAlertInput,
   LiveStatsEvent,
   PocketBaseListResponse,
   UpdateAlertInput,
 } from "./beszel-types";
+import { createRealtimeMetricsTopic, PocketBaseSseParser } from "./pocketbase-realtime";
 
 const logger = createLogger({ module: "beszel-integration" });
 
 const escapeFilterValue = (value: string) => value.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+const realtimeRetryDelaysMs = [200, 300, 500, 1_000, 1_500, 2_000] as const;
+
+const supportsRealtimeMetrics = (version: string) => {
+  const [major = 0, minor = 0] = version.replace(/^v/, "").split(".").map(Number);
+  return major > 0 || minor >= 13;
+};
+
+const waitForRetryAsync = async (delayMs: number, signal: AbortSignal) =>
+  await new Promise<void>((resolve) => {
+    if (signal.aborted) return resolve();
+    const timeout = setTimeout(done, delayMs);
+    signal.addEventListener("abort", done, { once: true });
+    function done() {
+      clearTimeout(timeout);
+      signal.removeEventListener("abort", done);
+      resolve();
+    }
+  });
+
+interface BeszelRealtimeSnapshot {
+  stats?: BeszelSystemStats;
+  container?: BeszelContainerStats[];
+}
+
+export const normalizeRealtimeSnapshot = (
+  payload: BeszelRealtimeSnapshot,
+  systemId: string,
+  receivedAt = new Date(),
+): LiveStatsEvent[] => {
+  const created = receivedAt.toISOString();
+  const id = `realtime-${receivedAt.getTime()}`;
+  const events: LiveStatsEvent[] = [];
+
+  if (payload.stats && typeof payload.stats === "object" && !Array.isArray(payload.stats)) {
+    events.push({
+      type: "system_stats",
+      record: { id, system: systemId, stats: payload.stats, type: "1m", created, updated: created },
+    });
+  }
+  if (Array.isArray(payload.container)) {
+    events.push({
+      type: "container_stats",
+      record: { id, system: systemId, stats: payload.container, type: "1m", created, updated: created },
+    });
+  }
+
+  return events;
+};
 
 interface BeszelSession {
   token: string;
@@ -285,123 +336,163 @@ export class BeszelIntegration extends Integration {
     });
   }
 
-  /**
-   * Subscribe to real-time system and container stats via PocketBase SSE.
-   * Connects to the PocketBase Realtime API and parses incoming SSE events,
-   * filtering by systemId. This provides live-updating metrics (≤1s latency
-   * from agent) instead of polling the REST API every 5s.
-   */
+  /** Subscribe to Beszel's custom one-second `rt_metrics` PocketBase topic. */
   public async subscribeRealtimeMetrics(
     systemId: string,
     onMessage: (event: LiveStatsEvent) => void,
     signal: AbortSignal,
   ): Promise<void> {
-    const session = await this.authenticateAsync();
-    const realtimeUrl = this.url("/api/realtime");
-
-    logger.debug("Opening Beszel SSE connection for realtime metrics", {
-      integrationId: this.integration.id,
-      systemId,
-      url: realtimeUrl.pathname,
-    });
-
-    const response = await fetchWithTrustedCertificatesAsync(realtimeUrl, {
-      headers: { Authorization: session.token },
-      signal,
-    });
-
-    if (!response.ok) {
-      logger.warn("Beszel SSE connection failed", {
-        integrationId: this.integration.id,
-        systemId,
-        status: response.status,
-      });
-      throw new ResponseError(response);
+    const systems = await this.getSystemsAsync();
+    const system = systems.find((candidate) => candidate.id === systemId);
+    if (!system) throw new Error("Selected Beszel system was not found");
+    if (system.status !== "up")
+      throw new Error(`Beszel system is ${system.status}; Live mode requires an online system`);
+    if (!supportsRealtimeMetrics(system.info.v)) {
+      throw new Error(`Beszel agent ${system.info.v} does not support Live mode; version 0.13.0 or newer is required`);
     }
 
-    const reader = response.body?.getReader();
-    if (!reader) throw new Error("No response body in SSE stream");
-
-    const decoder = new TextDecoder();
-    let buffer = "";
-    let subscribed = false;
-
-    const processLine = async (line: string) => {
-      if (!line.startsWith("data: ")) return;
+    const realtimeUrl = this.url("/api/realtime");
+    const topic = createRealtimeMetricsTopic(systemId);
+    let retryAttempt = 0;
+    let emittedSnapshotCount = 0;
 
+    while (!signal.aborted) {
       try {
-        const parsed = JSON.parse(line.slice(6)) as Record<string, unknown>;
-
-        // PB_CONNECT delivers the clientId — use it to POST our subscriptions
-        if (!subscribed && typeof parsed.clientId === "string") {
-          subscribed = true;
-          const subscribeResponse = await fetchWithTrustedCertificatesAsync(realtimeUrl, {
-            method: "POST",
+        let session = await this.authenticateAsync();
+        const openStream = async () =>
+          await fetchWithTrustedCertificatesAsync(realtimeUrl, {
             headers: {
+              Accept: "text/event-stream",
               Authorization: session.token,
-              "Content-Type": "application/json",
+              "Cache-Control": "no-cache",
+              Pragma: "no-cache",
             },
-            body: JSON.stringify({
-              clientId: parsed.clientId,
-              subscriptions: ["system_stats", "container_stats"],
-            }),
             signal,
+            bodyTimeout: 0,
           });
 
-          if (!subscribeResponse.ok) {
-            logger.warn("Beszel SSE subscription POST failed", {
+        logger.debug("Opening Beszel rt_metrics SSE connection", {
+          integrationId: this.integration.id,
+          systemId,
+          retryAttempt,
+        });
+        let response = await openStream();
+        if (response.status === 401) {
+          await this.sessionStore.clearAsync();
+          session = await this.authenticateAsync();
+          response = await openStream();
+        }
+        if (!response.ok) throw new ResponseError(response);
+
+        const reader = response.body?.getReader();
+        if (!reader) throw new Error("Beszel realtime response has no body");
+        const decoder = new TextDecoder();
+        const parser = new PocketBaseSseParser();
+        let clientId: string | undefined;
+
+        const processFrameAsync = async (frame: ReturnType<PocketBaseSseParser["push"]>[number]) => {
+          if (frame.event === "PB_CONNECT") {
+            let fallbackClientId: string | undefined;
+            try {
+              const connectData = JSON.parse(frame.data) as { clientId?: string };
+              fallbackClientId = connectData.clientId;
+            } catch {
+              // PocketBase normally provides the client ID in the SSE id field.
+            }
+            clientId = frame.id || fallbackClientId;
+            if (!clientId) throw new Error("PocketBase PB_CONNECT event did not include a client ID");
+
+            const subscribeResponse = await fetchWithTrustedCertificatesAsync(realtimeUrl, {
+              method: "POST",
+              headers: { Authorization: session.token, "Content-Type": "application/json" },
+              body: JSON.stringify({ clientId, subscriptions: [topic] }),
+              signal,
+            });
+            if (subscribeResponse.status === 401) await this.sessionStore.clearAsync();
+            if (!subscribeResponse.ok) throw new ResponseError(subscribeResponse);
+            logger.debug("Subscribed to Beszel rt_metrics topic", {
               integrationId: this.integration.id,
               systemId,
-              status: subscribeResponse.status,
+              clientId,
             });
-            throw new ResponseError(subscribeResponse);
+            return;
           }
 
-          logger.debug("Beszel SSE subscribed to collections", {
+          if (!frame.event.startsWith("rt_metrics")) return;
+          try {
+            const snapshot = JSON.parse(frame.data) as BeszelRealtimeSnapshot;
+            const events = normalizeRealtimeSnapshot(snapshot, systemId);
+            if (events.length === 0) return;
+            emittedSnapshotCount += 1;
+            retryAttempt = 0;
+            for (const event of events) onMessage(event);
+            if (emittedSnapshotCount === 1 || emittedSnapshotCount % 30 === 0) {
+              logger.debug("Forwarded Beszel rt_metrics snapshots", {
+                integrationId: this.integration.id,
+                systemId,
+                emittedSnapshotCount,
+                containerCount: snapshot.container?.length ?? 0,
+              });
+            }
+          } catch (error) {
+            logger.warn("Failed to parse Beszel rt_metrics payload", {
+              integrationId: this.integration.id,
+              systemId,
+              error: error instanceof Error ? error.message : String(error),
+            });
+          }
+        };
+
+        logger.debug("Beszel rt_metrics SSE connection established", {
+          integrationId: this.integration.id,
+          systemId,
+          contentType: response.headers.get("content-type"),
+        });
+        try {
+          while (!signal.aborted) {
+            const { done, value } = await reader.read();
+            if (done) break;
+            for (const frame of parser.push(decoder.decode(value, { stream: true }))) {
+              await processFrameAsync(frame);
+            }
+          }
+          for (const frame of parser.push(decoder.decode())) await processFrameAsync(frame);
+          for (const frame of parser.finish()) await processFrameAsync(frame);
+        } finally {
+          await reader.cancel().catch(() => {});
+          reader.releaseLock();
+        }
+
+        if (!signal.aborted) throw new Error("Beszel rt_metrics SSE stream ended unexpectedly");
+      } catch (error) {
+        if (signal.aborted) break;
+        if (error instanceof ResponseError && error.statusCode === 401 && retryAttempt === 0) {
+          retryAttempt += 1;
+          logger.debug("Retrying Beszel rt_metrics handshake after reauthentication", {
             integrationId: this.integration.id,
             systemId,
-            clientId: parsed.clientId,
           });
-          return;
+          continue;
         }
-
-        const record = parsed.record as Record<string, unknown> | undefined;
-        if (!record || record.system !== systemId) return;
-
-        if (Array.isArray(record.stats)) {
-          onMessage({
-            type: "container_stats",
-            record: record as unknown as BeszelContainerStatsRecord,
-          });
-        } else if (record.stats && typeof record.stats === "object") {
-          onMessage({
-            type: "system_stats",
-            record: record as unknown as BeszelSystemStatsRecord,
-          });
-        }
-      } catch (error) {
-        if (error instanceof ResponseError) throw error;
-        // skip malformed SSE data lines
+        if (error instanceof ResponseError && error.statusCode < 500 && error.statusCode !== 429) throw error;
+        const delayMs = realtimeRetryDelaysMs[Math.min(retryAttempt, realtimeRetryDelaysMs.length - 1)] ?? 2_000;
+        retryAttempt += 1;
+        logger.warn("Beszel rt_metrics connection lost; retrying", {
+          integrationId: this.integration.id,
+          systemId,
+          retryAttempt,
+          delayMs,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        await waitForRetryAsync(delayMs, signal);
       }
-    };
-
-    try {
-      while (true) {
-        const { done, value } = await reader.read();
-        if (done) break;
-
-        buffer += decoder.decode(value, { stream: true });
-        const lines = buffer.split("\n");
-        buffer = lines.pop() ?? "";
-
-        for (const line of lines) {
-          await processLine(line);
-        }
-      }
-    } finally {
-      reader.releaseLock();
-      logger.debug("Beszel SSE connection closed", { integrationId: this.integration.id, systemId });
     }
+
+    logger.debug("Beszel rt_metrics subscription stopped", {
+      integrationId: this.integration.id,
+      systemId,
+      emittedSnapshotCount,
+    });
   }
 
   protected async testingAsync(input: IntegrationTestingInput): Promise<TestingResult> {

packages/integrations/src/beszel/beszel-types.ts

@@ -133,9 +133,9 @@ export interface BeszelSystemStats {
   /** disk usage (%) */
   dp: number;
   /** disk read (MB/s) */
-  dr: number;
+  dr?: number;
   /** disk write (MB/s) */
-  dw: number;
+  dw?: number;
   /** disk read max (MB/s) */
   drm?: number;
   /** disk write max (MB/s) */
@@ -145,9 +145,9 @@ export interface BeszelSystemStats {
   /** disk IOPS max [read, write] */
   diom?: [number, number];
   /** network sent — all interfaces (bytes/s) */
-  ns: number;
+  ns?: number;
   /** network received — all interfaces (bytes/s) */
-  nr: number;
+  nr?: number;
   /** bandwidth — public interfaces only (bytes/s) [sent, recv]. Prefer over ns/nr */
   b?: [number, number];
   /** network sent max (bytes/s) */
@@ -169,26 +169,26 @@ export interface BeszelSystemStats {
 }
 
 export interface BeszelExtraFsStats {
-  /** total disk (bytes) */
+  /** total disk (GiB) */
   d: number;
-  /** disk used (bytes) */
+  /** disk used (GiB) */
   du: number;
   /** read (bytes/s) */
   r: number;
   /** write (bytes/s) */
   w: number;
   /** read max (bytes/s) */
-  rm: number;
+  rm?: number;
   /** write max (bytes/s) */
-  wm: number;
+  wm?: number;
   /** read (IOPS) */
-  rb: number;
+  rb?: number;
   /** write (IOPS) */
-  wb: number;
+  wb?: number;
   /** read max (IOPS) */
-  rbm: number;
+  rbm?: number;
   /** write max (IOPS) */
-  wbm: number;
+  wbm?: number;
 }
 
 export interface BeszelGPUData {
@@ -382,6 +382,8 @@ export interface BeszelSystemRow {
   memory: number;
   /** disk usage (%) */
   disk: number;
+  /** extra filesystem usage (%) keyed by mount path */
+  extraFilesystems: Record<string, number>;
   /** GPU usage (%) */
   gpu: number;
   /** load average [1m, 5m, 15m] */

packages/integrations/src/beszel/pocketbase-realtime.ts

@@ -0,0 +1,65 @@
+export interface PocketBaseSseFrame {
+  event: string;
+  id?: string;
+  data: string;
+}
+
+export const createRealtimeMetricsTopic = (systemId: string) =>
+  `rt_metrics?options=${encodeURIComponent(JSON.stringify({ query: { system: systemId } }))}`;
+
+export class PocketBaseSseParser {
+  private buffer = "";
+  private event = "message";
+  private id: string | undefined;
+  private dataLines: string[] = [];
+
+  public push(chunk: string): PocketBaseSseFrame[] {
+    this.buffer += chunk;
+    const frames: PocketBaseSseFrame[] = [];
+    const lines = this.buffer.split(/\r?\n/);
+    this.buffer = lines.pop() ?? "";
+
+    for (const line of lines) {
+      const frame = this.processLine(line);
+      if (frame) frames.push(frame);
+    }
+
+    return frames;
+  }
+
+  public finish(): PocketBaseSseFrame[] {
+    const frames: PocketBaseSseFrame[] = [];
+    if (this.buffer !== "") {
+      const frame = this.processLine(this.buffer);
+      if (frame) frames.push(frame);
+      this.buffer = "";
+    }
+    const trailingFrame = this.dispatch();
+    if (trailingFrame) frames.push(trailingFrame);
+    return frames;
+  }
+
+  private processLine(line: string): PocketBaseSseFrame | null {
+    if (line === "") return this.dispatch();
+    if (line.startsWith(":")) return null;
+
+    const colonIndex = line.indexOf(":");
+    const field = colonIndex === -1 ? line : line.slice(0, colonIndex);
+    let value = colonIndex === -1 ? "" : line.slice(colonIndex + 1);
+    if (value.startsWith(" ")) value = value.slice(1);
+
+    if (field === "event") this.event = value;
+    else if (field === "id") this.id = value;
+    else if (field === "data") this.dataLines.push(value);
+    return null;
+  }
+
+  private dispatch(): PocketBaseSseFrame | null {
+    if (this.dataLines.length === 0 && this.event === "message" && this.id === undefined) return null;
+    const frame = { event: this.event, id: this.id, data: this.dataLines.join("\n") };
+    this.event = "message";
+    this.id = undefined;
+    this.dataLines = [];
+    return frame;
+  }
+}

packages/integrations/src/beszel/test/beszel-integration.spec.ts

@@ -40,7 +40,124 @@ vi.mock("@homarr/core/infrastructure/certificates", () => ({
   getAllTrustedCertificatesAsync: vi.fn().mockResolvedValue([]),
 }));
 
-import { BeszelIntegration } from "../beszel-integration";
+import { BeszelIntegration, normalizeRealtimeSnapshot } from "../beszel-integration";
+import type { LiveStatsEvent } from "../beszel-types";
+import { createRealtimeMetricsTopic } from "../pocketbase-realtime";
+
+describe("normalizeRealtimeSnapshot", () => {
+  test("normalizes current Beszel host and container snapshots", () => {
+    const receivedAt = new Date("2026-07-11T13:51:30.217Z");
+    const events = normalizeRealtimeSnapshot(
+      {
+        stats: {
+          cpu: 18.07,
+          m: 31.09,
+          mu: 16.53,
+          mp: 53.18,
+          mb: 14.32,
+          s: 4,
+          su: 3.9,
+          d: 938.8,
+          du: 455.81,
+          dp: 51.15,
+          efs: { sda: { d: 1906.8, du: 1656.45, r: 25.63, w: 0 } },
+        },
+        container: [{ n: "homarr", c: 1.65, m: 650.56, b: [356066, 19702867] }],
+      },
+      "system-1",
+      receivedAt,
+    );
+
+    expect(events).toHaveLength(2);
+    expect(events[0]).toMatchObject({
+      type: "system_stats",
+      record: { system: "system-1", created: receivedAt.toISOString(), stats: { cpu: 18.07 } },
+    });
+    expect(events[1]).toMatchObject({
+      type: "container_stats",
+      record: { system: "system-1", stats: [{ n: "homarr", m: 650.56 }] },
+    });
+  });
+
+  test("ignores snapshots without metric data", () => {
+    expect(normalizeRealtimeSnapshot({}, "system-1")).toEqual([]);
+  });
+});
+
+describe("subscribeRealtimeMetrics", () => {
+  test("subscribes to Beszel rt_metrics and forwards one-second snapshots", async () => {
+    const originalFetch = globalThis.fetch;
+    const topic = createRealtimeMetricsTopic("system-1");
+    const subscriptionBodies: unknown[] = [];
+    const encoder = new TextEncoder();
+
+    globalThis.fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
+      const url = String(input);
+      if (url.endsWith("/api/collections/users/auth-with-password")) {
+        return Response.json({ token: "test-token", record: { id: "user-1" } });
+      }
+      if (url.includes("/api/collections/systems/records")) {
+        return Response.json({
+          page: 1,
+          perPage: 500,
+          totalItems: 1,
+          totalPages: 1,
+          items: [
+            {
+              id: "system-1",
+              name: "Server",
+              host: "localhost",
+              port: "45876",
+              status: "up",
+              info: { cpu: 1, mp: 2, dp: 3, u: 4, v: "0.18.7" },
+              created: "2026-07-11T00:00:00.000Z",
+              updated: "2026-07-11T00:00:00.000Z",
+            },
+          ],
+        });
+      }
+      if (url.endsWith("/api/realtime") && init?.method === "POST") {
+        subscriptionBodies.push(JSON.parse(String(init.body)));
+        expect(new Headers(init.headers).get("Authorization")).toBe("test-token");
+        return new Response(null, { status: 204 });
+      }
+      if (url.endsWith("/api/realtime")) {
+        const snapshot = JSON.stringify({
+          stats: { cpu: 5, m: 10, mu: 4, mp: 40, mb: 1, s: 0, su: 0, d: 20, du: 8, dp: 40 },
+          container: [{ n: "homarr", c: 1, m: 256 }],
+        });
+        const stream = new ReadableStream<Uint8Array>({
+          start(controller) {
+            controller.enqueue(encoder.encode("event: PB_CONNECT\nid: client-1\ndata: {}\n\n"));
+            controller.enqueue(encoder.encode(`event: ${topic}\ndata: ${snapshot}\n\n`));
+            controller.close();
+          },
+        });
+        return new Response(stream, { headers: { "Content-Type": "text/event-stream" } });
+      }
+      throw new Error(`Unexpected request: ${init?.method ?? "GET"} ${url}`);
+    }) as typeof fetch;
+
+    try {
+      const integration = createBeszelIntegration();
+      const controller = new AbortController();
+      const events: LiveStatsEvent[] = [];
+      await integration.subscribeRealtimeMetrics(
+        "system-1",
+        (event) => {
+          events.push(event);
+          if (events.length === 2) controller.abort();
+        },
+        controller.signal,
+      );
+
+      expect(subscriptionBodies).toEqual([{ clientId: "client-1", subscriptions: [topic] }]);
+      expect(events.map((event) => event.type)).toEqual(["system_stats", "container_stats"]);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});
 
 function createBeszelIntegration() {
   return new BeszelIntegration({

packages/integrations/src/beszel/test/pocketbase-realtime.spec.ts

@@ -0,0 +1,31 @@
+import { describe, expect, test } from "vitest";
+
+import { createRealtimeMetricsTopic, PocketBaseSseParser } from "../pocketbase-realtime";
+
+describe("PocketBaseSseParser", () => {
+  test("parses PB_CONNECT client IDs and CRLF-delimited frames split across chunks", () => {
+    const parser = new PocketBaseSseParser();
+    expect(parser.push("event: PB_CONNECT\r\nid: client-")).toEqual([]);
+    expect(parser.push("123\r\ndata: {}\r\n\r\n")).toEqual([{ event: "PB_CONNECT", id: "client-123", data: "{}" }]);
+  });
+
+  test("joins multiline data and ignores keepalive comments", () => {
+    const parser = new PocketBaseSseParser();
+    expect(parser.push(': keepalive\n\nevent: rt_metrics\ndata: {"stats":\ndata: {}}\n\n')).toEqual([
+      { event: "rt_metrics", id: undefined, data: '{"stats":\n{}}' },
+    ]);
+  });
+
+  test("flushes a trailing frame when the stream ends without a blank line", () => {
+    const parser = new PocketBaseSseParser();
+    expect(parser.push("event: rt_metrics\ndata: {}")).toEqual([]);
+    expect(parser.finish()).toEqual([{ event: "rt_metrics", id: undefined, data: "{}" }]);
+  });
+});
+
+describe("createRealtimeMetricsTopic", () => {
+  test("matches the PocketBase custom-topic options encoding used by Beszel", () => {
+    const topic = createRealtimeMetricsTopic("system-1");
+    expect(topic).toBe(`rt_metrics?options=${encodeURIComponent(JSON.stringify({ query: { system: "system-1" } }))}`);
+  });
+});

packages/integrations/src/mock/data/beszel.ts

@@ -226,6 +226,7 @@ export class BeszelMockService {
         u: randInt(86400, 2_000_000),
         mp: clamp(sys.baseMem + rand(-5, 5), 0, 100),
         dp: clamp(sys.baseDisk + rand(-2, 2), 0, 100),
+        efs: i === 0 ? { "/mnt/sda": 86.9, "/mnt/sdb": 70.5, "/mnt/sdc": 42.7 } : undefined,
         bb: rand(200, 3000) * KB,
         v: "0.8.2",
         g: i === 3 ? rand(15, 45) : 0,