fix(beszel): harden cache layer and prevent blank widgets

- Add debug logging across Beszel integration, request handler, and tRPC
  router (auth, fetch, SSE, per-procedure entry/exit) gated behind
  LOG_LEVEL=debug
- cached-request-handler: retry with backoff, isValid predicate to skip
  caching invalid data, in-flight dedup (single-flight per channel) to
  avoid thundering herd after cache invalidation
- redis channel: set-then-publish so storage is always consistent even
  when pubsub publish fails
- Beszel handlers: opt into fallbackToStaleOnError and retry so brief
  upstream hiccups don't throw away good cached data
- beszel-system-stats: add refetchInterval matching each time period's
  cache TTL so historical charts refresh in the background
- tRPC router: switch Promise.all -> Promise.allSettled for getSystems
  and getAlerts; getSystemStats returns an error object instead of
  throwing. Each result carries an optional error field so one failed
  integration no longer nukes the whole widget
- Widgets: add explicit isPending loader, empty state for grid and
  stats, and a shared per-integration error indicator (tooltip with
  failed integration names). Alerts and table now only show their
  empty state after loading completes
- Translation: add empty.noSystems key to beszelSystemGrid and
  beszelSystemStats

Addresses #6030

packages/widgets/src/beszel-alerts/component.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { useMemo } from "react";
-import { Badge, Box, Divider, Group, ScrollArea, Stack, Text, ThemeIcon } from "@mantine/core";
+import { Badge, Box, Center, Divider, Group, Loader, ScrollArea, Stack, Text, ThemeIcon } from "@mantine/core";
 import { IconBellOff, IconCircleCheck, IconFlame, IconHistory } from "@tabler/icons-react";
 import dayjs from "dayjs";
 import relativeTime from "dayjs/plugin/relativeTime";
@@ -12,6 +12,7 @@ import { clientApi } from "@homarr/api/client";
 import { useScopedI18n } from "@homarr/translation/client";
 
 import type { WidgetComponentProps } from "../definition";
+import { BeszelIntegrationErrorIndicator } from "../beszel/_shared/error-indicator";
 
 const alertIconMap: Record<string, LucideIcon> = {
   CPU: Cpu,
@@ -40,7 +41,7 @@ export default function BeszelAlertsWidget({
     () => ({ integrationIds, includeHistory: options.showHistory, maxHistoryItems: options.maxHistoryItems }),
     [integrationIds, options.showHistory, options.maxHistoryItems],
   );
-  const { data: results = [], error: alertsError } = clientApi.widget.beszel.getAlerts.useQuery(alertsInput, {
+  const { data: results = [], error: alertsError, isPending } = clientApi.widget.beszel.getAlerts.useQuery(alertsInput, {
     staleTime: 15_000,
     refetchInterval: 15_000,
     retry: false,
@@ -93,19 +94,31 @@ export default function BeszelAlertsWidget({
 
   if (alertsError) throw alertsError;
 
+  if (isPending) {
+    return (
+      <Center h="100%">
+        <Loader size="sm" />
+      </Center>
+    );
+  }
+
   return (
-    <ScrollArea h="100%" style={{ pointerEvents: isEditMode ? "none" : undefined }}>
-      <Stack gap="sm" p="sm">
-        {alerts.length === 0 && (
-          <Stack align="center" justify="center" py="xl" gap="xs">
-            <ThemeIcon variant="light" color="gray" size="lg" radius="xl">
-              <IconBellOff size={18} />
-            </ThemeIcon>
-            <Text size="sm" c="dimmed">
-              {t("empty")}
-            </Text>
-          </Stack>
-        )}
+    <Box h="100%" pos="relative">
+      <Box pos="absolute" top={4} right={8} style={{ zIndex: 1 }}>
+        <BeszelIntegrationErrorIndicator results={results} />
+      </Box>
+      <ScrollArea h="100%" style={{ pointerEvents: isEditMode ? "none" : undefined }}>
+        <Stack gap="sm" p="sm">
+          {alerts.length === 0 && (
+            <Stack align="center" justify="center" py="xl" gap="xs">
+              <ThemeIcon variant="light" color="gray" size="lg" radius="xl">
+                <IconBellOff size={18} />
+              </ThemeIcon>
+              <Text size="sm" c="dimmed">
+                {t("empty")}
+              </Text>
+            </Stack>
+          )}
 
         {triggeredAlerts.length > 0 && (
           <Stack gap={6}>
@@ -200,6 +213,7 @@ export default function BeszelAlertsWidget({
         )}
       </Stack>
     </ScrollArea>
+    </Box>
   );
 }
 

packages/widgets/src/beszel-system-grid/component.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import type { MantineSize } from "@mantine/core";
-import { Badge, Box, Card, Group, Progress, Text, Stack } from "@mantine/core";
+import { Badge, Box, Card, Center, Group, Loader, Progress, Text, Stack } from "@mantine/core";
 import {
   Activity,
   Battery,
@@ -15,6 +15,7 @@ import {
   Thermometer,
   Wifi,
 } from "lucide-react";
+import { IconServerOff } from "@tabler/icons-react";
 
 import { clientApi } from "@homarr/api/client";
 import { useRequiredBoard } from "@homarr/boards/context";
@@ -28,6 +29,7 @@ import type { BeszelSystemRow } from "../beszel/_shared/types";
 import { statusColorMap, thresholdColor } from "../beszel/_shared/colors";
 import { formatByteRate, formatLoadAvg, formatPercent, formatTemp, formatUptime } from "../beszel/_shared/format";
 import { useBeszelFilteredSystems, useBeszelSystemsSubscription } from "../beszel/_shared/hooks";
+import { BeszelIntegrationErrorIndicator } from "../beszel/_shared/error-indicator";
 import { BeszelSystemStatsModal } from "../beszel/_shared/system-stats-modal";
 
 interface SizeConfig {
@@ -363,7 +365,7 @@ export default function BeszelSystemGridWidget({
   const board = useRequiredBoard();
   const { openModal } = useModalAction(BeszelSystemStatsModal);
 
-  const { data: results = [], error: systemsError } = clientApi.widget.beszel.getSystems.useQuery(
+  const { data: results = [], error: systemsError, isPending } = clientApi.widget.beszel.getSystems.useQuery(
     { integrationIds },
     { staleTime: 5_000, refetchInterval: 5_000, retry: false },
   );
@@ -373,6 +375,30 @@ export default function BeszelSystemGridWidget({
 
   if (systemsError) throw systemsError;
 
+  if (isPending) {
+    return (
+      <Center h="100%">
+        <Loader size="sm" />
+      </Center>
+    );
+  }
+
+  if (filteredSystems.length === 0) {
+    return (
+      <Box h="100%" pos="relative" style={{ pointerEvents: isEditMode ? "none" : undefined }}>
+        <BeszelIntegrationErrorIndicator results={results} />
+        <Center h="100%">
+          <Stack align="center" gap="xs">
+            <IconServerOff size={28} opacity={0.5} />
+            <Text size="sm" c="dimmed">
+              {t("empty.noSystems")}
+            </Text>
+          </Stack>
+        </Center>
+      </Box>
+    );
+  }
+
   const cols = getColCount(width, filteredSystems.length);
   const rows = Math.ceil(filteredSystems.length / cols) || 1;
   const rawCellHeight = height / rows;
@@ -383,35 +409,37 @@ export default function BeszelSystemGridWidget({
   const maxMetrics = getMaxVisibleMetrics(effectiveCellHeight, size);
 
   return (
-    <Box
-      h="100%"
-      style={{
-        pointerEvents: isEditMode ? "none" : undefined,
-        display: "grid",
-        gridTemplateColumns: `repeat(${cols}, 1fr)`,
-        gridTemplateRows: scrollEnabled ? `repeat(${rows}, ${MIN_CELL_HEIGHT}px)` : `repeat(${rows}, 1fr)`,
-        gap: size.gap,
-        overflow: scrollEnabled ? "auto" : "hidden",
-      }}
-    >
-      {filteredSystems.map((system) => {
-        const integrationId = system._key.split(":")[0] ?? "";
-        const handleClick = isEditMode
-          ? undefined
-          : () => openModal({ integrationId, systemId: system.id }, { title: system.name });
-        return (
-          <SystemCard
-            key={system._key}
-            system={system}
-            options={options}
-            t={t}
-            size={size}
-            maxMetrics={maxMetrics}
-            itemRadius={board.itemRadius}
-            onClick={handleClick}
-          />
-        );
-      })}
+    <Box h="100%" pos="relative" style={{ pointerEvents: isEditMode ? "none" : undefined }}>
+      <BeszelIntegrationErrorIndicator results={results} />
+      <Box
+        h="100%"
+        style={{
+          display: "grid",
+          gridTemplateColumns: `repeat(${cols}, 1fr)`,
+          gridTemplateRows: scrollEnabled ? `repeat(${rows}, ${MIN_CELL_HEIGHT}px)` : `repeat(${rows}, 1fr)`,
+          gap: size.gap,
+          overflow: scrollEnabled ? "auto" : "hidden",
+        }}
+      >
+        {filteredSystems.map((system) => {
+          const integrationId = system._key.split(":")[0] ?? "";
+          const handleClick = isEditMode
+            ? undefined
+            : () => openModal({ integrationId, systemId: system.id }, { title: system.name });
+          return (
+            <SystemCard
+              key={system._key}
+              system={system}
+              options={options}
+              t={t}
+              size={size}
+              maxMetrics={maxMetrics}
+              itemRadius={board.itemRadius}
+              onClick={handleClick}
+            />
+          );
+        })}
+      </Box>
     </Box>
   );
 }

packages/widgets/src/beszel-system-stats/component.tsx

@@ -1,8 +1,8 @@
 "use client";
 
 import { useCallback, useMemo } from "react";
-import { Button, Center, Menu, ScrollArea, Select, SimpleGrid, Stack, Text } from "@mantine/core";
-import { IconPlugConnectedX, IconServer, IconQuestionMark } from "@tabler/icons-react";
+import { Button, Box, Center, Loader, Menu, ScrollArea, Select, SimpleGrid, Stack, Text } from "@mantine/core";
+import { IconPlugConnectedX, IconServer, IconQuestionMark, IconServerOff } from "@tabler/icons-react";
 
 import { clientApi } from "@homarr/api/client";
 import { useSession } from "@homarr/auth/client";
@@ -30,6 +30,7 @@ import {
   formatPercent,
   formatStorageBytes,
 } from "../beszel/_shared/format";
+import { BeszelIntegrationErrorIndicator } from "../beszel/_shared/error-indicator";
 import { makeTooltipProps } from "../beszel/_shared/tooltip";
 import { useLiveStats } from "../beszel/_shared/use-live-stats";
 
@@ -98,13 +99,18 @@ export default function BeszelSystemStatsWidget({
       timePeriod: options.timePeriod as "1m" | "1h" | "12h" | "24h" | "1w" | "30d",
       includeDocker: showDocker,
     },
-    { staleTime: 30 * 1000, enabled: !isLive && systemReady && selectedSystem !== "", retry: false },
+    {
+      staleTime: 30 * 1000,
+      refetchInterval: isLive ? false : 5_000,
+      enabled: !isLive && systemReady && selectedSystem !== "",
+      retry: false,
+    },
   );
 
   const { data: liveData, error: liveError } = useLiveStats(integrationIds, selectedSystem, isLive && systemReady);
 
   let activeStats:
-    | { systemStats: BeszelSystemStatsRecord[]; containerStats: BeszelContainerStatsRecord[] }
+    | { systemStats: BeszelSystemStatsRecord[]; containerStats: BeszelContainerStatsRecord[]; error?: string }
     | null
     | undefined = statsResult;
   if (isLive) {
@@ -191,6 +197,30 @@ export default function BeszelSystemStatsWidget({
   if (systemsError) throw systemsError;
   if (!isLive && statsError) throw statsError;
 
+  if (systemsPending) {
+    return (
+      <Center h="100%">
+        <Loader size="sm" />
+      </Center>
+    );
+  }
+
+  if (systems.length === 0) {
+    return (
+      <Box h="100%" pos="relative">
+        <BeszelIntegrationErrorIndicator results={systemsResult} />
+        <Center h="100%">
+          <Stack align="center" gap="xs">
+            <IconServerOff size={28} opacity={0.5} />
+            <Text size="sm" c="dimmed">
+              {t("empty.noSystems")}
+            </Text>
+          </Stack>
+        </Center>
+      </Box>
+    );
+  }
+
   if (!systemsPending && !systemExists && systems.length > 0) {
     return (
       <Center h="100%">
@@ -228,41 +258,62 @@ export default function BeszelSystemStatsWidget({
   }
 
   return (
-    <ScrollArea
-      h="100%"
-      className={classes.beszelStatsContainer}
-      style={{ pointerEvents: (isEditMode && "none") || undefined }}
-    >
-      <Stack gap="md" p="sm">
-        {!isEditMode && systems.length > 1 && (
-          <Menu position="bottom-start" withArrow shadow="md" withinPortal>
-            <Menu.Target>
-              <Button
-                variant="default"
-                size="compact-xs"
-                leftSection={<IconServer size={14} />}
-                className={classes.beszelStatsSystemToggle}
-              >
-                {selectedLabel}
-              </Button>
-            </Menu.Target>
-            <Menu.Dropdown>
-              {systems.map((s) => (
-                <Menu.Item
-                  key={s.value}
-                  fz="xs"
-                  fw={s.value === selectedSystem ? 600 : 400}
-                  c={s.value !== selectedSystem ? "dimmed" : undefined}
-                  onClick={() => handleSelectSystem(s.value)}
+    <Box h="100%" pos="relative">
+      <Box pos="absolute" top={4} right={8} style={{ zIndex: 1 }}>
+        <BeszelIntegrationErrorIndicator results={systemsResult} />
+      </Box>
+      <ScrollArea
+        h="100%"
+        className={classes.beszelStatsContainer}
+        style={{ pointerEvents: (isEditMode && "none") || undefined }}
+      >
+        <Stack gap="md" p="sm">
+          {!isEditMode && systems.length > 1 && (
+            <Menu position="bottom-start" withArrow shadow="md" withinPortal>
+              <Menu.Target>
+                <Button
+                  variant="default"
+                  size="compact-xs"
+                  leftSection={<IconServer size={14} />}
+                  className={classes.beszelStatsSystemToggle}
                 >
-                  {s.label}
-                </Menu.Item>
-              ))}
-            </Menu.Dropdown>
-          </Menu>
-        )}
+                  {selectedLabel}
+                </Button>
+              </Menu.Target>
+              <Menu.Dropdown>
+                {systems.map((s) => (
+                  <Menu.Item
+                    key={s.value}
+                    fz="xs"
+                    fw={s.value === selectedSystem ? 600 : 400}
+                    c={s.value !== selectedSystem ? "dimmed" : undefined}
+                    onClick={() => handleSelectSystem(s.value)}
+                  >
+                    {s.label}
+                  </Menu.Item>
+                ))}
+              </Menu.Dropdown>
+            </Menu>
+          )}
 
-        {activeStats && (
+          {!isLive && !activeStats && systemReady && selectedSystem !== "" && (
+            <Center h={CHART_HEIGHT}>
+              <Loader size="sm" />
+            </Center>
+          )}
+
+          {!isLive && activeStats && "error" in activeStats && activeStats.error && (
+            <Center h={CHART_HEIGHT}>
+              <Stack align="center" gap="xs">
+                <IconServerOff size={24} opacity={0.5} />
+                <Text size="sm" c="dimmed">
+                  {activeStats.error}
+                </Text>
+              </Stack>
+            </Center>
+          )}
+
+          {activeStats && (!("error" in activeStats) || !activeStats.error) && (
           <SimpleGrid cols={cols} spacing="md">
             {options.showCpu && cpuData.length > 0 && (
               <BeszelChartPanel
@@ -387,5 +438,6 @@ export default function BeszelSystemStatsWidget({
         )}
       </Stack>
     </ScrollArea>
+    </Box>
   );
 }

packages/widgets/src/beszel-system-table/component.tsx

@@ -3,7 +3,7 @@
 import "./styles.css";
 
 import { useMemo, useState } from "react";
-import { Group, Indicator, Progress, Text } from "@mantine/core";
+import { Center, Group, Indicator, Loader, Progress, Text } from "@mantine/core";
 import type { DataTableColumn, DataTableSortStatus } from "mantine-datatable";
 import { DataTable } from "mantine-datatable";
 import {
@@ -28,6 +28,7 @@ import type { BeszelSystemRow } from "../beszel/_shared/types";
 import { loadAvgColor, statusColorMap, thresholdColor } from "../beszel/_shared/colors";
 import { formatByteRate, formatLoadAvg, formatPercent, formatTemp, formatUptime } from "../beszel/_shared/format";
 import { useBeszelFilteredSystems, useBeszelSystemsSubscription } from "../beszel/_shared/hooks";
+import { BeszelIntegrationErrorIndicator } from "../beszel/_shared/error-indicator";
 import { BeszelSystemStatsModal } from "../beszel/_shared/system-stats-modal";
 
 const directionMultiplier: Record<string, number> = { asc: 1, desc: -1 };
@@ -69,7 +70,7 @@ export default function BeszelSystemTableWidget({
 }: WidgetComponentProps<"beszelSystemTable">) {
   const t = useScopedI18n("widget.beszelSystemTable");
   const { openModal } = useModalAction(BeszelSystemStatsModal);
-  const { data: results = [], error: systemsError } = clientApi.widget.beszel.getSystems.useQuery(
+  const { data: results = [], error: systemsError, isPending } = clientApi.widget.beszel.getSystems.useQuery(
     { integrationIds },
     { staleTime: 5_000, refetchInterval: 5_000, retry: false },
   );
@@ -265,22 +266,35 @@ export default function BeszelSystemTableWidget({
 
   if (systemsError) throw systemsError;
 
+  if (isPending) {
+    return (
+      <Center h="100%">
+        <Loader size="sm" />
+      </Center>
+    );
+  }
+
   return (
-    <DataTable
-      style={{ pointerEvents: isEditMode ? "none" : undefined }}
-      withTableBorder={false}
-      borderRadius={0}
-      highlightOnHover
-      fz={size.fontSize}
-      records={sortedSystems}
-      columns={columns}
-      sortStatus={sortStatus}
-      onSortStatusChange={setSortStatus}
-      noRecordsText={t("noRecords")}
-      idAccessor="_key"
-      height="100%"
-      className="beszel-table"
-      onRowClick={isEditMode ? undefined : handleRowClick}
-    />
+    <div style={{ position: "relative", height: "100%" }}>
+      <div style={{ position: "absolute", top: 4, right: 8, zIndex: 1 }}>
+        <BeszelIntegrationErrorIndicator results={results} />
+      </div>
+      <DataTable
+        style={{ pointerEvents: isEditMode ? "none" : undefined }}
+        withTableBorder={false}
+        borderRadius={0}
+        highlightOnHover
+        fz={size.fontSize}
+        records={sortedSystems}
+        columns={columns}
+        sortStatus={sortStatus}
+        onSortStatusChange={setSortStatus}
+        noRecordsText={t("noRecords")}
+        idAccessor="_key"
+        height="100%"
+        className="beszel-table"
+        onRowClick={isEditMode ? undefined : handleRowClick}
+      />
+    </div>
   );
 }

packages/widgets/src/beszel/_shared/error-indicator.tsx

@@ -0,0 +1,29 @@
+"use client";
+
+import { Tooltip } from "@mantine/core";
+import { IconAlertTriangle } from "@tabler/icons-react";
+
+interface IntegrationResult {
+  integrationId: string;
+  integrationName?: string;
+  error?: string;
+}
+
+interface BeszelIntegrationErrorIndicatorProps {
+  results: IntegrationResult[];
+}
+
+export function BeszelIntegrationErrorIndicator({ results }: BeszelIntegrationErrorIndicatorProps) {
+  const failed = results.filter((r) => "error" in r && r.error);
+  if (failed.length === 0) return null;
+  const label = failed.map((r) => r.integrationName ?? r.integrationId).join(", ");
+  return (
+    <Tooltip label={label} position="left" withArrow>
+      <IconAlertTriangle
+        size={14}
+        color="var(--mantine-color-orange-6)"
+        style={{ cursor: "help", flexShrink: 0 }}
+      />
+    </Tooltip>
+  );
+}