fix(beszel): harden cache layer and prevent blank widgets (#6052)

* fix(beszel): harden cache layer and prevent blank widgets

- Add debug logging across Beszel integration, request handler, and tRPC
  router (auth, fetch, SSE, per-procedure entry/exit) gated behind
  LOG_LEVEL=debug
- cached-request-handler: retry with backoff, isValid predicate to skip
  caching invalid data, in-flight dedup (single-flight per channel) to
  avoid thundering herd after cache invalidation
- redis channel: set-then-publish so storage is always consistent even
  when pubsub publish fails
- Beszel handlers: opt into fallbackToStaleOnError and retry so brief
  upstream hiccups don't throw away good cached data
- beszel-system-stats: add refetchInterval matching each time period's
  cache TTL so historical charts refresh in the background
- tRPC router: switch Promise.all -> Promise.allSettled for getSystems
  and getAlerts; getSystemStats returns an error object instead of
  throwing. Each result carries an optional error field so one failed
  integration no longer nukes the whole widget
- Widgets: add explicit isPending loader, empty state for grid and
  stats, and a shared per-integration error indicator (tooltip with
  failed integration names). Alerts and table now only show their
  empty state after loading completes
- Translation: add empty.noSystems key to beszelSystemGrid and
  beszelSystemStats

Addresses #6030

* style: auto-fix lint and format

---------

Co-authored-by: ajnart <ajnart@users.noreply.github.com>

packages/request-handler/src/beszel.ts

@@ -1,5 +1,6 @@
 import dayjs from "dayjs";
 
+import { createLogger } from "@homarr/core/infrastructure/logs";
 import { createIntegrationAsync } from "@homarr/integrations";
 import type {
   BeszelAlert,
@@ -13,6 +14,8 @@ import type {
 
 import { createCachedIntegrationRequestHandler } from "./lib/cached-integration-request-handler";
 
+const logger = createLogger({ module: "beszelRequestHandler" });
+
 export type { BeszelSystemRow } from "@homarr/integrations/types";
 
 /**
@@ -56,6 +59,7 @@ export const beszelSystemsRequestHandler = createCachedIntegrationRequestHandler
   Record<string, never>
 >({
   async requestAsync(integration) {
+    const start = performance.now();
     const instance = await createIntegrationAsync(integration);
     const systems = await instance.getSystemsAsync();
     const enriched = await Promise.all(
@@ -64,9 +68,16 @@ export const beszelSystemsRequestHandler = createCachedIntegrationRequestHandler
         return mapToSystemRow(system, details);
       }),
     );
+    logger.debug("beszelSystems fetch completed", {
+      integrationId: integration.id,
+      durationMs: Math.round(performance.now() - start),
+      count: enriched.length,
+    });
     return enriched;
   },
   cacheDuration: dayjs.duration(5, "seconds"),
+  fallbackToStaleOnError: true,
+  retry: { attempts: 2, delayMs: 500 },
   queryKey: "beszelSystems",
 });
 
@@ -81,12 +92,21 @@ export const beszelAlertsRequestHandler = createCachedIntegrationRequestHandler<
   { includeHistory: boolean; maxHistoryItems: number }
 >({
   async requestAsync(integration, input) {
+    const start = performance.now();
     const instance = await createIntegrationAsync(integration);
     const alerts = await instance.getAlertsAsync();
     const history = input.includeHistory ? await instance.getAlertHistoryAsync(undefined, input.maxHistoryItems) : [];
+    logger.debug("beszelAlerts fetch completed", {
+      integrationId: integration.id,
+      durationMs: Math.round(performance.now() - start),
+      alertsCount: alerts.length,
+      historyCount: history.length,
+    });
     return { alerts, history };
   },
   cacheDuration: dayjs.duration(15, "seconds"),
+  fallbackToStaleOnError: true,
+  retry: { attempts: 2, delayMs: 500 },
   queryKey: "beszelAlerts",
 });
 
@@ -110,15 +130,26 @@ export const beszelStatsRequestHandler = createCachedIntegrationRequestHandler<
   { systemId: string; timePeriod: string; includeDocker: boolean }
 >({
   async requestAsync(integration, input) {
+    const start = performance.now();
     const config = timePeriodConfig[input.timePeriod] ?? { type: "1m", perPage: 60, cacheSeconds: 60 };
     const instance = await createIntegrationAsync(integration);
     const systemStats = await instance.getSystemStatsAsync(input.systemId, config.type, config.perPage);
     const containerStats = input.includeDocker
       ? await instance.getContainerStatsAsync(input.systemId, config.type, config.perPage).catch(() => [])
       : [];
+    logger.debug("beszelStats fetch completed", {
+      integrationId: integration.id,
+      systemId: input.systemId,
+      timePeriod: input.timePeriod,
+      durationMs: Math.round(performance.now() - start),
+      systemStatsCount: systemStats.length,
+      containerStatsCount: containerStats.length,
+    });
     return { systemStats, containerStats };
   },
   cacheDuration: dayjs.duration(60, "seconds"),
+  fallbackToStaleOnError: true,
+  retry: { attempts: 2, delayMs: 500 },
   queryKey: "beszelStats",
   cacheDurationForInput(input) {
     const config = timePeriodConfig[input.timePeriod];

packages/request-handler/src/lib/cached-integration-request-handler.ts

@@ -18,6 +18,9 @@ interface Options<TData, TKind extends IntegrationKind, TInput extends Record<st
   requestAsync: (integration: IntegrationOfKind<TKind>, input: TInput) => Promise<TData>;
   cacheDuration: Duration;
   cacheDurationForInput?: (input: TInput) => Duration | undefined;
+  fallbackToStaleOnError?: boolean;
+  retry?: { attempts?: number; delayMs?: number };
+  isValid?: (data: TData) => boolean;
 }
 
 export const createCachedIntegrationRequestHandler = <
@@ -35,6 +38,9 @@ export const createCachedIntegrationRequestHandler = <
           return await options.requestAsync(input.integration, input.options);
         },
         cacheDuration: options.cacheDurationForInput?.(itemOptions) ?? options.cacheDuration,
+        fallbackToStaleOnError: options.fallbackToStaleOnError,
+        retry: options.retry,
+        isValid: options.isValid,
         createRedisChannel(input, handlerOptions) {
           return createIntegrationOptionsChannel<TData>(input.integration.id, handlerOptions.queryKey, input.options);
         },

packages/request-handler/src/lib/cached-request-handler.ts

@@ -7,6 +7,16 @@ import type { createChannelWithLatestAndEvents } from "@homarr/redis";
 
 const logger = createLogger({ module: "cachedRequestHandler" });
 
+type FetchResult<TData> = { data: TData; timestamp: Date };
+type InFlightEntry = Promise<FetchResult<unknown>>;
+
+const inFlightFetches = new Map<string, InFlightEntry>();
+
+const isAbortedError = (error: unknown): boolean => {
+  if (!(error instanceof Error)) return false;
+  return error.name === "AbortError" || error.name === "TimeoutError";
+};
+
 interface Options<TData, TInput extends Record<string, unknown>> {
   // Unique key for this request handler
   queryKey: string;
@@ -17,28 +27,70 @@ interface Options<TData, TInput extends Record<string, unknown>> {
   ) => ReturnType<typeof createChannelWithLatestAndEvents<TData>>;
   cacheDuration: Duration;
   fallbackToStaleOnError?: boolean;
+  // Retry the upstream call on failure. attempts=1 means no retry (default).
+  // AbortError / TimeoutError are never retried.
+  retry?: { attempts?: number; delayMs?: number };
+  // Predicate to decide whether a fetched response is worth caching.
+  // Return false to skip the cache write (next call will refetch).
+  isValid?: (data: TData) => boolean;
 }
 
+const defaultRetry = { attempts: 1, delayMs: 0 };
+const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
 export const createCachedRequestHandler = <TData, TInput extends Record<string, unknown>>(
   options: Options<TData, TInput>,
 ) => {
   return {
     handler: (input: TInput) => {
       const channel = options.createRedisChannel(input, options);
+      const retryConfig = { ...defaultRetry, ...options.retry };
 
       return {
         async getCachedOrUpdatedDataAsync({ forceUpdate = false }) {
           const channelData = await channel.getAsync();
 
-          const requestNewDataAsync = async () => {
-            try {
-              const data = await options.requestAsync(input);
-              await channel.publishAndUpdateLastStateAsync(data);
-              return {
-                data,
-                timestamp: new Date(),
-              };
-            } catch (error) {
+          const requestNewDataAsync = (): Promise<FetchResult<TData>> => {
+            const inflight = inFlightFetches.get(channel.name);
+            if (inflight) {
+              logger.debug("Cached request handler joining in-flight fetch", {
+                channel: channel.name,
+                queryKey: options.queryKey,
+              });
+              return inflight as Promise<FetchResult<TData>>;
+            }
+
+            const promise = (async (): Promise<FetchResult<TData>> => {
+              let lastError: unknown;
+              for (let attempt = 1; attempt <= retryConfig.attempts; attempt++) {
+                try {
+                  const data = await options.requestAsync(input);
+                  if (options.isValid && !options.isValid(data)) {
+                    logger.warn("Cached request handler received invalid data, not caching", {
+                      channel: channel.name,
+                      queryKey: options.queryKey,
+                      attempt,
+                    });
+                    return { data, timestamp: new Date() };
+                  }
+                  await channel.publishAndUpdateLastStateAsync(data);
+                  return { data, timestamp: new Date() };
+                } catch (error) {
+                  lastError = error;
+                  if (isAbortedError(error) || attempt === retryConfig.attempts) {
+                    break;
+                  }
+                  logger.warn("Cached request handler fetch failed, retrying", {
+                    channel: channel.name,
+                    queryKey: options.queryKey,
+                    attempt,
+                    nextDelayMs: retryConfig.delayMs,
+                    error: error instanceof Error ? `${error.name}: ${error.message}` : String(error),
+                  });
+                  await sleep(retryConfig.delayMs);
+                }
+              }
+
               if (options.fallbackToStaleOnError && channelData) {
                 logger.warn(
                   new ErrorWithMetadata(
@@ -46,15 +98,21 @@ export const createCachedRequestHandler = <TData, TInput extends Record<string,
                     {
                       channel: channel.name,
                       queryKey: options.queryKey,
+                      attempts: retryConfig.attempts,
                     },
-                    { cause: error },
+                    { cause: lastError },
                   ),
                 );
                 return channelData;
               }
 
-              throw error;
-            }
+              throw lastError;
+            })().finally(() => {
+              inFlightFetches.delete(channel.name);
+            });
+
+            inFlightFetches.set(channel.name, promise as InFlightEntry);
+            return promise;
           };
 
           if (forceUpdate) {