fix(beszel): prevent live subscription memory growth (#6448)

* fix(beszel): bound live stats subscriptions

* fix(beszel): avoid blocking subscription teardown

* fix(import): map new clock options
This commit is contained in:
Thomas authored and GitHub committed 2026-07-23 02:03:08 +02:00
1 parent 9168f8114b
commit 14c3fa2e76
9 files changed
+286 -104

No files matched your search

@@ -0,0 +1,45 @@
import { describe, expect, test } from "vitest";
import { BoundedAsyncQueue } from "../bounded-async-queue";
describe("BoundedAsyncQueue", () => {
test("keeps only the newest values when the queue is full", async () => {
const queue = new BoundedAsyncQueue<number>(2);
queue.push(1);
queue.push(2);
queue.push(3);
queue.close();
await expect(queue.next()).resolves.toEqual({ value: 2, done: false });
await expect(queue.next()).resolves.toEqual({ value: 3, done: false });
await expect(queue.next()).resolves.toEqual({ value: undefined, done: true });
});
test("resolves a pending read as soon as a value arrives", async () => {
const queue = new BoundedAsyncQueue<string>(1);
const next = queue.next();
queue.push("latest");
await expect(next).resolves.toEqual({ value: "latest", done: false });
});
test("resolves a pending read when the producer closes", async () => {
const queue = new BoundedAsyncQueue<number>(1);
const next = queue.next();
queue.close();
await expect(next).resolves.toEqual({ value: undefined, done: true });
});
test("rejects a pending read when the producer fails", async () => {
const queue = new BoundedAsyncQueue<number>(1);
const next = queue.next();
const error = new Error("upstream failed");
queue.fail(error);
await expect(next).rejects.toBe(error);
});
});
+76 -79
View File
@@ -1,5 +1,4 @@
import { TRPCError } from "@trpc/server";
import { observable } from "@trpc/server/observable";
import { z } from "zod/v4";
import { createLogger } from "@homarr/core/infrastructure/logs";
@@ -14,8 +13,10 @@ import {
import { settleIntegrationQueries } from "../../settle-integrations";
import { createManyIntegrationMiddleware } from "../../middlewares/integration";
import { createTRPCRouter, publicProcedure } from "../../trpc";
import { BoundedAsyncQueue } from "./bounded-async-queue";
const logger = createLogger({ module: "beszelRouter" });
const MAX_PENDING_LIVE_EVENTS = 4;
const errorMessage = (error: unknown) => (error instanceof Error ? error.message : String(error));
@@ -203,98 +204,94 @@ export const beszelRouter = createTRPCRouter({
systemId: z.string(),
}),
)
.subscription(({ ctx, input }) => {
return observable<LiveStatsEvent>((emit) => {
const controller = new AbortController();
let isActive = true;
let emittedEventCount = 0;
.subscription(async function* ({ ctx, input, signal }) {
const integration = ctx.integrations[0];
if (!integration) {
throw new TRPCError({ code: "BAD_REQUEST", message: "At least one Beszel integrationId is required" });
}
logger.debug("Beszel realtime subscription started", {
userId: ctx.session?.user?.id,
integrationIds: ctx.integrations.map((integration) => integration.id),
systemId: input.systemId,
});
const queue = new BoundedAsyncQueue<LiveStatsEvent>(MAX_PENDING_LIVE_EVENTS);
const controller = new AbortController();
let emittedEventCount = 0;
const stop = () => {
controller.abort();
void queue.return();
};
if (signal?.aborted) return;
signal?.addEventListener("abort", stop, { once: true });
logger.debug("Beszel realtime subscription started", {
userId: ctx.session?.user?.id,
integrationIds: ctx.integrations.map((candidate) => candidate.id),
systemId: input.systemId,
});
try {
const instance = await createIntegrationAsync(integration);
if (controller.signal.aborted) return;
void (async () => {
const integration = ctx.integrations[0];
if (!integration) {
emit.error(
new TRPCError({ code: "BAD_REQUEST", message: "At least one Beszel integrationId is required" }),
);
return;
}
try {
const instance = await createIntegrationAsync(integration);
if (typeof instance.subscribeRealtimeMetrics === "function") {
await instance.subscribeRealtimeMetrics(
input.systemId,
(event) => {
if (!isActive) return;
emittedEventCount += 1;
if (emittedEventCount <= 2 || emittedEventCount % 60 === 0) {
logger.debug("Forwarding Beszel realtime events", {
userId: ctx.session?.user?.id,
integrationId: integration.id,
systemId: input.systemId,
eventType: event.type,
emittedEventCount,
statsCount: Array.isArray(event.record.stats) ? event.record.stats.length : undefined,
});
}
emit.next(event);
},
controller.signal,
);
} else {
emit.error(
new TRPCError({
code: "METHOD_NOT_SUPPORTED",
message: `Integration ${integration.kind} does not support realtime metrics`,
}),
);
}
} catch (error) {
if (isActive) {
logger.warn("Beszel realtime subscription failed", {
userId: ctx.session?.user?.id,
integrationId: integration.id,
systemId: input.systemId,
emittedEventCount,
error: errorMessage(error),
if (typeof instance.subscribeRealtimeMetrics !== "function") {
throw new TRPCError({
code: "METHOD_NOT_SUPPORTED",
message: `Integration ${integration.kind} does not support realtime metrics`,
});
emit.error(
error instanceof TRPCError
? error
: new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message: error instanceof Error ? error.message : String(error),
}),
);
}
}
if (isActive) {
logger.debug("Beszel realtime subscription completed", {
await instance.subscribeRealtimeMetrics(
input.systemId,
(event) => {
emittedEventCount += 1;
if (emittedEventCount <= 2 || emittedEventCount % 60 === 0) {
logger.debug("Forwarding Beszel realtime events", {
userId: ctx.session?.user?.id,
integrationId: integration.id,
systemId: input.systemId,
eventType: event.type,
emittedEventCount,
statsCount: Array.isArray(event.record.stats) ? event.record.stats.length : undefined,
});
}
queue.push(event);
},
controller.signal,
);
queue.close();
} catch (error) {
if (controller.signal.aborted) {
queue.close();
return;
}
logger.warn("Beszel realtime subscription failed", {
userId: ctx.session?.user?.id,
integrationId: integration.id,
systemId: input.systemId,
emittedEventCount,
error: errorMessage(error),
});
emit.complete();
queue.fail(
error instanceof TRPCError
? error
: new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message: error instanceof Error ? error.message : String(error),
}),
);
}
})();
return () => {
isActive = false;
logger.debug("Beszel realtime subscription cancelled", {
userId: ctx.session?.user?.id,
integrationIds: ctx.integrations.map((integration) => integration.id),
systemId: input.systemId,
emittedEventCount,
});
controller.abort();
};
});
for await (const event of queue) yield event;
} finally {
stop();
signal?.removeEventListener("abort", stop);
logger.debug("Beszel realtime subscription stopped", {
userId: ctx.session?.user?.id,
integrationId: integration.id,
systemId: input.systemId,
emittedEventCount,
});
}
}),
});
@@ -0,0 +1,88 @@
type QueueResult<T> = IteratorResult<T, undefined>;
type PendingRead<T> = {
resolve: (result: QueueResult<T>) => void;
reject: (reason?: unknown) => void;
};
/**
* A small async queue for live streams.
*
* Live metrics are snapshots, so retaining every snapshot while a client is
* slow is both unnecessary and dangerous. When the queue is full, discard
* the oldest pending snapshot and keep the newest one.
*/
export class BoundedAsyncQueue<T> implements AsyncIterableIterator<T> {
private readonly values: T[] = [];
private pendingRead: PendingRead<T> | undefined;
private closed = false;
private failed = false;
private failure: unknown;
public constructor(private readonly maxSize: number) {
if (!Number.isInteger(maxSize) || maxSize < 1) {
throw new Error("A bounded async queue requires a positive integer max size");
}
}
public push(value: T): void {
if (this.closed) return;
if (this.pendingRead) {
const pendingRead = this.pendingRead;
this.pendingRead = undefined;
pendingRead.resolve({ value, done: false });
return;
}
if (this.values.length >= this.maxSize) this.values.shift();
this.values.push(value);
}
public close(): void {
if (this.closed) return;
this.closed = true;
if (this.pendingRead) {
const pendingRead = this.pendingRead;
this.pendingRead = undefined;
pendingRead.resolve({ value: undefined, done: true });
}
}
public fail(error: unknown): void {
if (this.closed) return;
this.closed = true;
this.failed = true;
this.failure = error;
this.values.length = 0;
if (this.pendingRead) {
const pendingRead = this.pendingRead;
this.pendingRead = undefined;
pendingRead.reject(error);
}
}
public next(): Promise<QueueResult<T>> {
if (this.values.length > 0) {
return Promise.resolve({ value: this.values.shift() as T, done: false });
}
if (this.failed) return Promise.reject(this.failure);
if (this.closed) return Promise.resolve({ value: undefined, done: true });
return new Promise<QueueResult<T>>((resolve, reject) => {
this.pendingRead = { resolve, reject };
});
}
public return(): Promise<QueueResult<T>> {
this.values.length = 0;
this.close();
return Promise.resolve({ value: undefined, done: true });
}
public [Symbol.asyncIterator](): AsyncIterableIterator<T> {
return this;
}
}
@@ -67,12 +67,14 @@ export const fetchWithTrustedCertificatesAsync = async (
url: RequestInfo,
options?: RequestInit & { timeout?: number; bodyTimeout?: number },
): Promise<Response> => {
const agent = await createCertificateAgentAsync(
undefined,
options?.bodyTimeout !== undefined ? { bodyTimeout: options.bodyTimeout } : undefined,
);
const agent =
options?.dispatcher ??
(await createCertificateAgentAsync(
undefined,
options?.bodyTimeout !== undefined ? { bodyTimeout: options.bodyTimeout } : undefined,
));
if (options?.timeout) {
const { bodyTimeout: _bodyTimeout, ...fetchOptions } = options;
const { bodyTimeout: _bodyTimeout, dispatcher: _dispatcher, ...fetchOptions } = options;
return await withTimeoutAsync(
async (signal) =>
fetch(url, {
@@ -84,7 +86,7 @@ export const fetchWithTrustedCertificatesAsync = async (
);
}
const { bodyTimeout: _bodyTimeout, ...fetchOptions } = options ?? {};
const { bodyTimeout: _bodyTimeout, dispatcher: _dispatcher, ...fetchOptions } = options ?? {};
return fetch(url, {
...fetchOptions,
dispatcher: agent,
@@ -1,6 +1,7 @@
import { ResponseError } from "@homarr/common/server";
import { createLogger } from "@homarr/core/infrastructure/logs";
import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http";
import { createCertificateAgentAsync, fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http";
import type { Response as UndiciResponse } from "undici";
import type { IntegrationInput, IntegrationTestingInput } from "../base/integration";
import { Integration } from "../base/integration";
@@ -357,19 +358,41 @@ export class BeszelIntegration extends Integration {
let emittedSnapshotCount = 0;
while (!signal.aborted) {
let streamResponse: UndiciResponse | undefined;
let streamAgent: Awaited<ReturnType<typeof createCertificateAgentAsync>> | undefined;
const closeStreamAsync = async () => {
const response = streamResponse;
streamResponse = undefined;
const agent = streamAgent;
streamAgent = undefined;
if (response?.body) await response.body.cancel().catch(() => {});
if (agent) await agent.close().catch(() => {});
};
try {
let session = await this.authenticateAsync();
const openStream = async () =>
await fetchWithTrustedCertificatesAsync(realtimeUrl, {
headers: {
Accept: "text/event-stream",
Authorization: session.token,
"Cache-Control": "no-cache",
Pragma: "no-cache",
},
signal,
bodyTimeout: 0,
});
const openStream = async () => {
streamAgent = await createCertificateAgentAsync(undefined, { bodyTimeout: 0 });
try {
const response = await fetchWithTrustedCertificatesAsync(realtimeUrl, {
headers: {
Accept: "text/event-stream",
Authorization: session.token,
"Cache-Control": "no-cache",
Pragma: "no-cache",
},
signal,
bodyTimeout: 0,
dispatcher: streamAgent,
});
streamResponse = response;
return response;
} catch (error) {
await closeStreamAsync();
throw error;
}
};
logger.debug("Opening Beszel rt_metrics SSE connection", {
integrationId: this.integration.id,
@@ -378,6 +401,7 @@ export class BeszelIntegration extends Integration {
});
let response = await openStream();
if (response.status === 401) {
await closeStreamAsync();
await this.sessionStore.clearAsync();
session = await this.authenticateAsync();
response = await openStream();
@@ -402,14 +426,21 @@ export class BeszelIntegration extends Integration {
clientId = frame.id || fallbackClientId;
if (!clientId) throw new Error("PocketBase PB_CONNECT event did not include a client ID");
if (!streamAgent) throw new Error("Beszel realtime stream agent is not available");
const subscribeResponse = await fetchWithTrustedCertificatesAsync(realtimeUrl, {
method: "POST",
headers: { Authorization: session.token, "Content-Type": "application/json" },
body: JSON.stringify({ clientId, subscriptions: [topic] }),
signal,
dispatcher: streamAgent,
});
if (subscribeResponse.status === 401) await this.sessionStore.clearAsync();
if (!subscribeResponse.ok) throw new ResponseError(subscribeResponse);
try {
if (subscribeResponse.status === 401) await this.sessionStore.clearAsync();
if (!subscribeResponse.ok) throw new ResponseError(subscribeResponse);
await subscribeResponse.arrayBuffer();
} finally {
if (subscribeResponse.body) await subscribeResponse.body.cancel().catch(() => {});
}
logger.debug("Subscribed to Beszel rt_metrics topic", {
integrationId: this.integration.id,
systemId,
@@ -461,10 +492,12 @@ export class BeszelIntegration extends Integration {
} finally {
await reader.cancel().catch(() => {});
reader.releaseLock();
await closeStreamAsync();
}
if (!signal.aborted) throw new Error("Beszel rt_metrics SSE stream ended unexpectedly");
} catch (error) {
await closeStreamAsync();
if (signal.aborted) break;
if (error instanceof ResponseError && error.statusCode === 401 && retryAttempt === 0) {
retryAttempt += 1;
@@ -10,6 +10,7 @@ vi.hoisted(() => {
const BESZEL_URL = process.env.BESZEL_TEST_URL ?? "http://localhost:8090";
const BESZEL_EMAIL = process.env.BESZEL_TEST_EMAIL ?? "";
const BESZEL_PASSWORD = process.env.BESZEL_TEST_PASSWORD ?? "";
const closeAgent = vi.hoisted(() => vi.fn().mockResolvedValue(undefined));
vi.mock("@homarr/redis", () => ({
createGetSetChannel: () => ({
@@ -32,7 +33,7 @@ vi.mock("@homarr/core/infrastructure/logs", () => ({
vi.mock("@homarr/core/infrastructure/http", () => ({
fetchWithTrustedCertificatesAsync: (url: URL | string, init?: RequestInit) => fetch(url, init),
createAxiosCertificateInstanceAsync: vi.fn().mockResolvedValue({}),
createCertificateAgentAsync: vi.fn().mockResolvedValue(undefined),
createCertificateAgentAsync: vi.fn().mockResolvedValue({ close: closeAgent }),
}));
vi.mock("@homarr/core/infrastructure/certificates", () => ({
@@ -153,6 +154,7 @@ describe("subscribeRealtimeMetrics", () => {
expect(subscriptionBodies).toEqual([{ clientId: "client-1", subscriptions: [topic] }]);
expect(events.map((event) => event.type)).toEqual(["system_stats", "container_stats"]);
expect(closeAgent).toHaveBeenCalled();
} finally {
globalThis.fetch = originalFetch;
}
@@ -65,6 +65,9 @@ const optionMapping: OptionMapping = {
useCustomTimezone: () => true,
customTimeFormat: () => undefined,
customDateFormat: () => undefined,
showWeather: () => undefined,
weatherLocation: () => undefined,
isWeatherFormatFahrenheit: () => undefined,
},
downloads: {
activeTorrentThreshold: (oldOptions) =>
@@ -1,6 +1,6 @@
"use client";
import { useCallback, useEffect, useRef, useState } from "react";
import { useCallback, useEffect, useRef, useState, useSyncExternalStore } from "react";
import { skipToken } from "@tanstack/react-query";
import { clientApi } from "@homarr/api/client";
@@ -10,10 +10,22 @@ import type { BeszelContainerStatsRecord, BeszelSystemStatsRecord } from "@homar
// without synthesizing points for missed updates.
const MAX_BUFFER = 60;
const subscribeToVisibility = (onChange: () => void) => {
if (typeof document === "undefined") return () => {};
document.addEventListener("visibilitychange", onChange);
return () => document.removeEventListener("visibilitychange", onChange);
};
const getPageVisibility = () => typeof document === "undefined" || document.visibilityState === "visible";
const usePageVisible = () => useSyncExternalStore(subscribeToVisibility, getPageVisibility, () => true);
export const useLiveStats = (integrationIds: string[], systemId: string, enabled: boolean) => {
const [systemStats, setSystemStats] = useState<BeszelSystemStatsRecord[]>([]);
const [containerStats, setContainerStats] = useState<BeszelContainerStatsRecord[]>([]);
const [error, setError] = useState<Error | null>(null);
const pageVisible = usePageVisible();
const subscriptionEnabled = enabled && pageVisible && systemId !== "";
// Append to buffer, trimming to MAX_BUFFER
const appendSystemStats = useCallback((record: BeszelSystemStatsRecord) => {
@@ -31,7 +43,7 @@ export const useLiveStats = (integrationIds: string[], systemId: string, enabled
}, []);
clientApi.widget.beszel.subscribeSystemStats.useSubscription(
enabled && systemId !== "" ? { integrationIds, systemId } : skipToken,
subscriptionEnabled ? { integrationIds, systemId } : skipToken,
{
onData(event) {
setError(null);
@@ -49,7 +61,7 @@ export const useLiveStats = (integrationIds: string[], systemId: string, enabled
// Reset buffers when system changes or integration changes
const prevKeyRef = useRef<string>("");
const currentKey = `${integrationIds.join(",")}:${systemId}:${enabled}`;
const currentKey = `${integrationIds.join(",")}:${systemId}:${subscriptionEnabled}`;
useEffect(() => {
if (prevKeyRef.current !== currentKey) {
prevKeyRef.current = currentKey;