fix(beszel): harden cache layer and prevent blank widgets (#6052)

* fix(beszel): harden cache layer and prevent blank widgets

- Add debug logging across Beszel integration, request handler, and tRPC
  router (auth, fetch, SSE, per-procedure entry/exit) gated behind
  LOG_LEVEL=debug
- cached-request-handler: retry with backoff, isValid predicate to skip
  caching invalid data, in-flight dedup (single-flight per channel) to
  avoid thundering herd after cache invalidation
- redis channel: set-then-publish so storage is always consistent even
  when pubsub publish fails
- Beszel handlers: opt into fallbackToStaleOnError and retry so brief
  upstream hiccups don't throw away good cached data
- beszel-system-stats: add refetchInterval matching each time period's
  cache TTL so historical charts refresh in the background
- tRPC router: switch Promise.all -> Promise.allSettled for getSystems
  and getAlerts; getSystemStats returns an error object instead of
  throwing. Each result carries an optional error field so one failed
  integration no longer nukes the whole widget
- Widgets: add explicit isPending loader, empty state for grid and
  stats, and a shared per-integration error indicator (tooltip with
  failed integration names). Alerts and table now only show their
  empty state after loading completes
- Translation: add empty.noSystems key to beszelSystemGrid and
  beszelSystemStats

Addresses #6030

* style: auto-fix lint and format

---------

Co-authored-by: ajnart <ajnart@users.noreply.github.com>

packages/request-handler/src/lib/cached-request-handler.ts

@@ -7,6 +7,16 @@ import type { createChannelWithLatestAndEvents } from "@homarr/redis";
 
 const logger = createLogger({ module: "cachedRequestHandler" });
 
+type FetchResult<TData> = { data: TData; timestamp: Date };
+type InFlightEntry = Promise<FetchResult<unknown>>;
+
+const inFlightFetches = new Map<string, InFlightEntry>();
+
+const isAbortedError = (error: unknown): boolean => {
+  if (!(error instanceof Error)) return false;
+  return error.name === "AbortError" || error.name === "TimeoutError";
+};
+
 interface Options<TData, TInput extends Record<string, unknown>> {
   // Unique key for this request handler
   queryKey: string;
@@ -17,28 +27,70 @@ interface Options<TData, TInput extends Record<string, unknown>> {
   ) => ReturnType<typeof createChannelWithLatestAndEvents<TData>>;
   cacheDuration: Duration;
   fallbackToStaleOnError?: boolean;
+  // Retry the upstream call on failure. attempts=1 means no retry (default).
+  // AbortError / TimeoutError are never retried.
+  retry?: { attempts?: number; delayMs?: number };
+  // Predicate to decide whether a fetched response is worth caching.
+  // Return false to skip the cache write (next call will refetch).
+  isValid?: (data: TData) => boolean;
 }
 
+const defaultRetry = { attempts: 1, delayMs: 0 };
+const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
 export const createCachedRequestHandler = <TData, TInput extends Record<string, unknown>>(
   options: Options<TData, TInput>,
 ) => {
   return {
     handler: (input: TInput) => {
       const channel = options.createRedisChannel(input, options);
+      const retryConfig = { ...defaultRetry, ...options.retry };
 
       return {
         async getCachedOrUpdatedDataAsync({ forceUpdate = false }) {
           const channelData = await channel.getAsync();
 
-          const requestNewDataAsync = async () => {
-            try {
-              const data = await options.requestAsync(input);
-              await channel.publishAndUpdateLastStateAsync(data);
-              return {
-                data,
-                timestamp: new Date(),
-              };
-            } catch (error) {
+          const requestNewDataAsync = (): Promise<FetchResult<TData>> => {
+            const inflight = inFlightFetches.get(channel.name);
+            if (inflight) {
+              logger.debug("Cached request handler joining in-flight fetch", {
+                channel: channel.name,
+                queryKey: options.queryKey,
+              });
+              return inflight as Promise<FetchResult<TData>>;
+            }
+
+            const promise = (async (): Promise<FetchResult<TData>> => {
+              let lastError: unknown;
+              for (let attempt = 1; attempt <= retryConfig.attempts; attempt++) {
+                try {
+                  const data = await options.requestAsync(input);
+                  if (options.isValid && !options.isValid(data)) {
+                    logger.warn("Cached request handler received invalid data, not caching", {
+                      channel: channel.name,
+                      queryKey: options.queryKey,
+                      attempt,
+                    });
+                    return { data, timestamp: new Date() };
+                  }
+                  await channel.publishAndUpdateLastStateAsync(data);
+                  return { data, timestamp: new Date() };
+                } catch (error) {
+                  lastError = error;
+                  if (isAbortedError(error) || attempt === retryConfig.attempts) {
+                    break;
+                  }
+                  logger.warn("Cached request handler fetch failed, retrying", {
+                    channel: channel.name,
+                    queryKey: options.queryKey,
+                    attempt,
+                    nextDelayMs: retryConfig.delayMs,
+                    error: error instanceof Error ? `${error.name}: ${error.message}` : String(error),
+                  });
+                  await sleep(retryConfig.delayMs);
+                }
+              }
+
               if (options.fallbackToStaleOnError && channelData) {
                 logger.warn(
                   new ErrorWithMetadata(
@@ -46,15 +98,21 @@ export const createCachedRequestHandler = <TData, TInput extends Record<string,
                     {
                       channel: channel.name,
                       queryKey: options.queryKey,
+                      attempts: retryConfig.attempts,
                     },
-                    { cause: error },
+                    { cause: lastError },
                   ),
                 );
                 return channelData;
               }
 
-              throw error;
-            }
+              throw lastError;
+            })().finally(() => {
+              inFlightFetches.delete(channel.name);
+            });
+
+            inFlightFetches.set(channel.name, promise as InFlightEntry);
+            return promise;
           };
 
           if (forceUpdate) {