feat: retain application log history

apps/docs/docs/management/logs/index.mdx

@@ -0,0 +1,34 @@
+---
+tags:
+  - Administration
+  - Logs
+  - Troubleshooting
+---
+
+# Logs
+
+The logs page shows recent Homarr application logs and continues updating as new events occur. Use it to investigate
+widget, integration, and server errors without opening the container logs separately.
+
+## Opening the logs page
+
+Navigate to **Management** > **Tools** > **Logs**. The page and widget log links are only available to users with the
+**View logs** permission.
+
+When a widget reports an error, **Check logs for more details** opens the logs page in a new tab and marks the time when
+the widget displayed the error. The related server error normally appears immediately above that marker. If those logs
+have already been removed from the retained history, the marker explains that they are no longer available.
+
+## Retained and live logs
+
+Homarr retains the latest 5,000 log events in Redis. Opening the page first loads this history and then continues with
+live events without a gap. The history is bounded, so older events are removed automatically.
+
+The history survives normal application restarts when the configured Redis instance persists its data. For complete
+platform-level history, startup failures, or events older than the retained limit, use the logs provided by Docker or
+your deployment platform.
+
+## Display controls
+
+The severity selector applies to both retained and live events. Selecting a different level redraws the existing view
+using that threshold. The font-size control changes the terminal text size and is remembered in the browser.

apps/nextjs/src/app/[locale]/manage/tools/logs/client.tsx

@@ -1,10 +1,15 @@
 "use client";
 
+import type { ComponentType } from "react";
 import dynamic from "next/dynamic";
 
+interface ClientSideTerminalComponentProps {
+  focusTimestamp?: number;
+}
+
 export const ClientSideTerminalComponent = dynamic(
   () => import("./terminal").then(({ TerminalComponent }) => TerminalComponent),
   {
     ssr: false,
   },
-);
+) as ComponentType<ClientSideTerminalComponentProps>;

apps/nextjs/src/app/[locale]/manage/tools/logs/page.tsx

@@ -29,12 +29,25 @@ export async function generateMetadata() {
   };
 }
 
-export default async function LogsManagementPage() {
+interface LogsManagementPageProps {
+  searchParams: Promise<{ focus?: string }>;
+}
+
+export default async function LogsManagementPage({ searchParams }: LogsManagementPageProps) {
   const session = await auth();
   if (!session?.user.permissions.includes("other-view-logs")) {
     notFound();
   }
 
+  const { focus } = await searchParams;
+  let focusTimestamp: number | undefined;
+  if (focus !== undefined) {
+    const parsedFocusTimestamp = Number(focus);
+    if (Number.isSafeInteger(parsedFocusTimestamp) && parsedFocusTimestamp > 0) {
+      focusTimestamp = parsedFocusTimestamp;
+    }
+  }
+
   return (
     <LogContextProvider defaultLevel={logsEnv.LEVEL}>
       <Group justify="space-between" align="center" wrap="nowrap">
@@ -45,7 +58,7 @@ export default async function LogsManagementPage() {
         </Group>
       </Group>
       <Box style={{ borderRadius: 6 }} h={fullHeightWithoutHeaderAndFooter} p="md" bg="black">
-        <ClientSideTerminalComponent />
+        <ClientSideTerminalComponent focusTimestamp={focusTimestamp} />
       </Box>
     </LogContextProvider>
   );

apps/nextjs/src/app/[locale]/manage/tools/logs/terminal.tsx

@@ -7,25 +7,176 @@ import { FitAddon } from "@xterm/addon-fit";
 import { Terminal } from "@xterm/xterm";
 
 import { clientApi } from "@homarr/api/client";
+import type { LoggerMessage, LogLevel } from "@homarr/core/infrastructure/logs/constants";
+import { LOG_HISTORY_MAX_ENTRIES, logLevels } from "@homarr/core/infrastructure/logs/constants";
+import { useI18n } from "@homarr/translation/client";
 
 import { useLogContext } from "./log-context";
 import classes from "./terminal.module.css";
 
-export const TerminalComponent = () => {
+const ALL_LOG_LEVELS = [...logLevels];
+const TERMINAL_SCROLLBACK_LINES = LOG_HISTORY_MAX_ENTRIES * 4;
+
+interface FocusMarker {
+  index: number;
+  text: string;
+}
+
+const getFocusMarker = (
+  messages: LoggerMessage[],
+  visibleMessages: LoggerMessage[],
+  focusTimestamp: number | undefined,
+  markerText: string,
+  expiredMarkerText: string,
+): FocusMarker | null => {
+  if (focusTimestamp === undefined) return null;
+
+  let oldestTimestamp = Number.POSITIVE_INFINITY;
+  for (const message of messages) {
+    oldestTimestamp = Math.min(oldestTimestamp, message.timestamp.getTime());
+  }
+  if (oldestTimestamp === Number.POSITIVE_INFINITY || focusTimestamp < oldestTimestamp) {
+    return { index: 0, text: expiredMarkerText };
+  }
+
+  let markerIndex = 0;
+  for (const [index, message] of visibleMessages.entries()) {
+    if (message.timestamp.getTime() <= focusTimestamp) {
+      markerIndex = index + 1;
+    }
+  }
+
+  return {
+    index: markerIndex,
+    text: markerText,
+  };
+};
+
+const findMarkerLine = (terminal: Terminal, markerText: string) => {
+  const buffer = terminal.buffer.active;
+  let logicalLine = "";
+  let logicalLineStart = 0;
+
+  for (let index = 0; index < buffer.length; index++) {
+    const line = buffer.getLine(index);
+    if (!line) continue;
+
+    if (!line.isWrapped) {
+      logicalLine = "";
+      logicalLineStart = index;
+    }
+    logicalLine += line.translateToString(true);
+
+    if (logicalLine.includes(markerText)) {
+      return logicalLineStart;
+    }
+  }
+
+  return null;
+};
+
+const renderMessages = ({
+  terminal,
+  messages,
+  activeLevels,
+  focusTimestamp,
+  markerText,
+  expiredMarkerText,
+}: {
+  terminal: Terminal;
+  messages: LoggerMessage[];
+  activeLevels: LogLevel[];
+  focusTimestamp: number | undefined;
+  markerText: string;
+  expiredMarkerText: string;
+}) => {
+  const activeLevelSet = new Set(activeLevels);
+  const visibleMessages = messages.filter((message) => activeLevelSet.has(message.level));
+  const focusMarker = getFocusMarker(messages, visibleMessages, focusTimestamp, markerText, expiredMarkerText);
+  const output = visibleMessages.flatMap((message, index) => {
+    if (focusMarker?.index === index) {
+      return [`\x1b[33m--- ${focusMarker.text} ---\x1b[0m`, message.message];
+    }
+    return [message.message];
+  });
+
+  if (focusMarker?.index === visibleMessages.length) {
+    output.push(`\x1b[33m--- ${focusMarker.text} ---\x1b[0m`);
+  }
+
+  terminal.reset();
+  if (output.length === 0) return;
+
+  terminal.write(`${output.join("\r\n")}\r\n`, () => {
+    if (!focusMarker) {
+      terminal.scrollToBottom();
+      return;
+    }
+
+    const markerLine = findMarkerLine(terminal, focusMarker.text);
+    if (markerLine !== null) {
+      terminal.scrollToLine(markerLine);
+    }
+  });
+};
+
+interface TerminalComponentProps {
+  focusTimestamp?: number;
+}
+
+export const TerminalComponent = ({ focusTimestamp }: TerminalComponentProps) => {
   const ref = useRef<HTMLDivElement>(null);
   const { activeLevels, fontSize } = useLogContext();
+  const t = useI18n();
 
   const terminalRef = useRef<Terminal>(null);
   const fitAddonRef = useRef<FitAddon>(null);
+  const messagesRef = useRef<LoggerMessage[]>([]);
+  const initialFontSizeRef = useRef(fontSize);
+  const activeLevelsRef = useRef(activeLevels);
+  const markerTextRef = useRef(t("log.context.widgetError"));
+  const expiredMarkerTextRef = useRef(t("log.context.expiredWidgetError"));
+  const focusTimestampRef = useRef(focusTimestamp);
+
+  activeLevelsRef.current = activeLevels;
+  markerTextRef.current = t("log.context.widgetError");
+  expiredMarkerTextRef.current = t("log.context.expiredWidgetError");
+  focusTimestampRef.current = focusTimestamp;
+
+  const redrawTerminal = () => {
+    const terminal = terminalRef.current;
+    if (!terminal) return;
+
+    renderMessages({
+      terminal,
+      messages: messagesRef.current,
+      activeLevels: activeLevelsRef.current,
+      focusTimestamp: focusTimestampRef.current,
+      markerText: markerTextRef.current,
+      expiredMarkerText: expiredMarkerTextRef.current,
+    });
+  };
 
   clientApi.log.subscribe.useSubscription(
     {
-      levels: activeLevels,
+      levels: ALL_LOG_LEVELS,
     },
     {
-      onData(data) {
-        terminalRef.current?.writeln(data.message);
-        terminalRef.current?.refresh(0, terminalRef.current.rows - 1);
+      onData(event) {
+        if (event.type === "history") {
+          messagesRef.current = event.messages.slice(-LOG_HISTORY_MAX_ENTRIES);
+          redrawTerminal();
+          return;
+        }
+
+        messagesRef.current.push(event.message);
+        if (messagesRef.current.length > LOG_HISTORY_MAX_ENTRIES) {
+          messagesRef.current.shift();
+        }
+
+        if (activeLevelsRef.current.includes(event.message.level)) {
+          terminalRef.current?.writeln(event.message.message);
+        }
       },
       onError(err) {
         alert(err);
@@ -44,22 +195,26 @@ export const TerminalComponent = () => {
       cursorBlink: false,
       disableStdin: true,
       convertEol: true,
-      fontSize,
+      fontSize: initialFontSizeRef.current,
+      scrollback: TERMINAL_SCROLLBACK_LINES,
     });
     terminalRef.current.open(ref.current);
     terminalRef.current.loadAddon(canvasAddon);
 
-    setTimeout(() => {
+    const fitTimeout = window.setTimeout(() => {
       const fitAddon = new FitAddon();
       fitAddonRef.current = fitAddon;
       terminalRef.current?.loadAddon(fitAddon);
       fitAddon.fit();
+      redrawTerminal();
     });
 
     return () => {
+      window.clearTimeout(fitTimeout);
       canvasAddon.dispose();
       terminalRef.current?.dispose();
       terminalRef.current = null;
+      fitAddonRef.current = null;
     };
   }, []);
 
@@ -69,5 +224,9 @@ export const TerminalComponent = () => {
     fitAddonRef.current?.fit();
   }, [fontSize]);
 
+  useEffect(() => {
+    redrawTerminal();
+  }, [activeLevels, focusTimestamp]);
+
   return <Box ref={ref} id="terminal" className={classes.outerTerminal} h="100%"></Box>;
 };

packages/api/src/router/log.ts

@@ -11,6 +11,10 @@ import { createTRPCRouter, permissionRequiredProcedure } from "../trpc";
 
 const logger = createLogger({ module: "logRouter" });
 
+type LogSubscriptionEvent =
+  | { type: "history"; messages: LoggerMessage[] }
+  | { type: "message"; message: LoggerMessage };
+
 export const logRouter = createTRPCRouter({
   subscribe: permissionRequiredProcedure
     .requiresPermission("other-view-logs")
@@ -20,15 +24,61 @@ export const logRouter = createTRPCRouter({
       }),
     )
     .subscription(({ input }) => {
-      return observable<LoggerMessage>((emit) => {
-        const unsubscribe = loggingChannel.subscribe((data) => {
-          if (!input.levels.includes(data.level)) return;
-          emit.next(data);
-        });
-        logger.info("A tRPC client has connected to the logging procedure");
+      return observable<LogSubscriptionEvent>((emit) => {
+        let isClosed = false;
+        let unsubscribe: (() => void) | null = null;
+        let isHistoryLoaded = false;
+        const bufferedMessages: LoggerMessage[] = [];
+
+        const onMessage = (message: LoggerMessage) => {
+          if (!input.levels.includes(message.level)) return;
+
+          if (!isHistoryLoaded) {
+            bufferedMessages.push(message);
+            return;
+          }
+
+          emit.next({ type: "message", message });
+        };
+
+        void loggingChannel
+          .subscribeAsync(onMessage)
+          .then(async (unsubscribeFromLogging) => {
+            if (isClosed) {
+              unsubscribeFromLogging();
+              return;
+            }
+
+            unsubscribe = unsubscribeFromLogging;
+            logger.info("A tRPC client has connected to the logging procedure");
+
+            const history = (await loggingChannel.getRecentAsync()).filter((message) =>
+              input.levels.includes(message.level),
+            );
+            if (isClosed) return;
+
+            emit.next({ type: "history", messages: history });
+
+            const historyIds = new Set(history.map((message) => message.id));
+            isHistoryLoaded = true;
+            for (const message of bufferedMessages) {
+              if (!historyIds.has(message.id)) {
+                emit.next({ type: "message", message });
+              }
+            }
+            bufferedMessages.length = 0;
+          })
+          .catch((error: unknown) => {
+            unsubscribe?.();
+            unsubscribe = null;
+            if (!isClosed) {
+              emit.error(error);
+            }
+          });
 
         return () => {
-          unsubscribe();
+          isClosed = true;
+          unsubscribe?.();
         };
       });
     }),

packages/core/src/infrastructure/logs/constants.ts

@@ -1,6 +1,17 @@
 export const logLevels = ["error", "warn", "info", "debug"] as const;
 export type LogLevel = (typeof logLevels)[number];
 
+export interface LoggerMessage {
+  id: string;
+  timestamp: Date;
+  message: string;
+  level: LogLevel;
+}
+
+export const LOG_HISTORY_MAX_ENTRIES = 5_000;
+export const LOG_HISTORY_KEY = "list:logging:history";
+export const LOG_PUBLISH_CHANNEL = "pubSub:logging";
+
 export const logLevelConfiguration = {
   error: {
     prefix: "🔴",

packages/core/src/infrastructure/logs/transports/redis-transport.ts

@@ -1,8 +1,11 @@
-import superjson from "superjson";
+import { randomUUID } from "node:crypto";
+import { stringify } from "superjson";
 import Transport from "winston-transport";
 
 import type { RedisClient } from "../../redis/client";
 import { createRedisClient } from "../../redis/client";
+import type { LoggerMessage } from "../constants";
+import { LOG_HISTORY_KEY, LOG_HISTORY_MAX_ENTRIES, LOG_PUBLISH_CHANNEL } from "../constants";
 
 const messageSymbol = Symbol.for("message");
 const levelSymbol = Symbol.for("level");
@@ -13,12 +16,11 @@ const levelSymbol = Symbol.for("level");
 //
 export class RedisTransport extends Transport {
   private redis: RedisClient | null = null;
-  public static readonly publishChannel = "pubSub:logging";
 
   /**
-   * Log the info to the Redis channel
+   * Retain the log in a bounded Redis list and publish it to live subscribers.
    */
-  log(info: { [messageSymbol]: string; [levelSymbol]: string }, callback: () => void) {
+  log(info: { [messageSymbol]: string; [levelSymbol]: LoggerMessage["level"] }, callback: () => void) {
     setImmediate(() => {
       this.emit("logged", info);
     });
@@ -26,19 +28,23 @@ export class RedisTransport extends Transport {
     // Is only initialized here because it did not work when initialized in the constructor or outside the class
     this.redis ??= createRedisClient();
 
+    const message: LoggerMessage = {
+      id: randomUUID(),
+      timestamp: new Date(),
+      message: info[messageSymbol],
+      level: info[levelSymbol],
+    };
+    const serializedMessage = stringify(message);
+
     this.redis
-      .publish(
-        RedisTransport.publishChannel,
-        superjson.stringify({
-          message: info[messageSymbol],
-          level: info[levelSymbol],
-        }),
-      )
-      .then(() => {
-        callback();
-      })
+      .multi()
+      .lpush(LOG_HISTORY_KEY, serializedMessage)
+      .ltrim(LOG_HISTORY_KEY, 0, LOG_HISTORY_MAX_ENTRIES - 1)
+      .publish(LOG_PUBLISH_CHANNEL, serializedMessage)
+      .exec()
       .catch(() => {
         // Ignore errors
-      });
+      })
+      .finally(callback);
   }
 }

packages/redis/src/index.ts

@@ -1,6 +1,10 @@
-import type { LogLevel } from "@homarr/core/infrastructure/logs/constants";
+import { parse } from "superjson";
+
+import type { LoggerMessage } from "@homarr/core/infrastructure/logs/constants";
+import { LOG_HISTORY_KEY, LOG_HISTORY_MAX_ENTRIES } from "@homarr/core/infrastructure/logs/constants";
 
 import { createListChannel, createSubPubChannel } from "./lib/channel";
+import { createRedisConnection } from "./lib/connection";
 
 export {
   handshakeAsync,
@@ -15,9 +19,16 @@ export const pingChannel = createSubPubChannel<
 >("ping");
 export const pingUrlChannel = createListChannel<string>("ping-url");
 
-export interface LoggerMessage {
-  message: string;
-  level: LogLevel;
-}
+export type { LoggerMessage } from "@homarr/core/infrastructure/logs/constants";
 
-export const loggingChannel = createSubPubChannel<LoggerMessage>("logging");
+const liveLoggingChannel = createSubPubChannel<LoggerMessage>("logging", { persist: false });
+const loggingHistoryClient = createRedisConnection();
+
+export const loggingChannel = {
+  subscribe: liveLoggingChannel.subscribe,
+  subscribeAsync: liveLoggingChannel.subscribeAsync,
+  getRecentAsync: async () => {
+    const messages = await loggingHistoryClient.lrange(LOG_HISTORY_KEY, 0, LOG_HISTORY_MAX_ENTRIES - 1);
+    return messages.map((message) => parse<LoggerMessage>(message)).toReversed();
+  },
+};

packages/redis/src/lib/channel-subscription-tracker.ts

@@ -18,6 +18,7 @@ type SubscriptionCallback = (message: string) => MaybePromise<void>;
  */
 export class ChannelSubscriptionTracker {
   private static subscriptions = new Map<string, Map<string, SubscriptionCallback>>();
+  private static subscriptionReadiness = new Map<string, Promise<void>>();
   private static redis = createRedisConnection();
   private static listenerActive = false;
 
@@ -28,6 +29,25 @@ export class ChannelSubscriptionTracker {
    * @returns a function to unsubscribe from the channel
    */
   public static subscribe(channelName: string, callback: SubscriptionCallback) {
+    return this.registerSubscription(channelName, callback).unsubscribe;
+  }
+
+  /**
+   * Subscribes to a channel and resolves once Redis confirms the subscription.
+   */
+  public static async subscribeAsync(channelName: string, callback: SubscriptionCallback) {
+    const subscription = this.registerSubscription(channelName, callback);
+
+    try {
+      await subscription.ready;
+      return subscription.unsubscribe;
+    } catch (error) {
+      subscription.unsubscribe();
+      throw error;
+    }
+  }
+
+  private static registerSubscription(channelName: string, callback: SubscriptionCallback) {
     logger.debug("Adding redis channel callback", { channel: channelName });
 
     // We only want to activate the listener once
@@ -38,11 +58,14 @@ export class ChannelSubscriptionTracker {
 
     const channelSubscriptions = this.subscriptions.get(channelName) ?? new Map<string, SubscriptionCallback>();
     const id = randomUUID();
+    let ready = this.subscriptionReadiness.get(channelName) ?? Promise.resolve();
 
     // If there are no subscriptions to the channel, subscribe to it
     if (channelSubscriptions.size === 0) {
       logger.debug("Subscribing to redis channel", { channel: channelName });
-      void this.redis.subscribe(channelName);
+      ready = this.redis.subscribe(channelName).then(() => undefined);
+      this.subscriptionReadiness.set(channelName, ready);
+      void ready.catch(() => undefined);
     }
 
     logger.debug("Adding redis channel callback", { channel: channelName, id });
@@ -50,24 +73,26 @@ export class ChannelSubscriptionTracker {
 
     this.subscriptions.set(channelName, channelSubscriptions);
 
-    // Return a function to unsubscribe
-    return () => {
+    const unsubscribe = () => {
       logger.debug("Removing redis channel callback", { channel: channelName, id });
 
-      const channelSubscriptions = this.subscriptions.get(channelName);
-      if (!channelSubscriptions) return;
+      const currentSubscriptions = this.subscriptions.get(channelName);
+      if (!currentSubscriptions) return;
 
-      channelSubscriptions.delete(id);
+      currentSubscriptions.delete(id);
 
       // If there are no subscriptions to the channel, unsubscribe from it
-      if (channelSubscriptions.size >= 1) {
+      if (currentSubscriptions.size >= 1) {
         return;
       }
 
       logger.debug("Unsubscribing from redis channel", { channel: channelName });
       void this.redis.unsubscribe(channelName);
       this.subscriptions.delete(channelName);
+      this.subscriptionReadiness.delete(channelName);
     };
+
+    return { ready, unsubscribe };
   }
 
   /**

packages/redis/src/lib/channel.ts

@@ -1,4 +1,4 @@
-import superjson from "superjson";
+import superjson, { parse } from "superjson";
 
 import { createId } from "@homarr/common";
 
@@ -8,6 +8,12 @@ import { createRedisConnection } from "./connection";
 const publisher = createRedisConnection();
 const lastDataClient = createRedisConnection();
 
+const deserializeCallback =
+  <TData>(callback: (data: TData) => void) =>
+  (message: string) => {
+    callback(parse(message));
+  };
+
 /**
  * Creates a new pub/sub channel.
  * @param name name of the channel
@@ -29,9 +35,19 @@ export const createSubPubChannel = <TData>(name: string, { persist }: { persist:
           }
         });
       }
-      return ChannelSubscriptionTracker.subscribe(channelName, (message) => {
-        callback(superjson.parse(message));
-      });
+      return ChannelSubscriptionTracker.subscribe(channelName, deserializeCallback(callback));
+    },
+    /**
+     * Subscribes to the channel and resolves once Redis confirms the subscription.
+     */
+    subscribeAsync: async (callback: (data: TData) => void) => {
+      if (persist) {
+        const data = await lastDataClient.get(lastChannelName);
+        if (data) {
+          callback(superjson.parse(data));
+        }
+      }
+      return await ChannelSubscriptionTracker.subscribeAsync(channelName, deserializeCallback(callback));
     },
     /**
      * Publish data to the channel with last data saved.

packages/translation/src/lang/en.json

@@ -8385,6 +8385,10 @@
     }
   },
   "log": {
+    "context": {
+      "widgetError": "Widget reported an error around this time",
+      "expiredWidgetError": "Logs around this widget error are no longer retained"
+    },
     "level": {
       "option": {
         "debug": "Debug",

packages/widgets/src/errors/base-component.tsx

@@ -1,5 +1,7 @@
+import { useState } from "react";
 import { Anchor, Button, Stack, Text } from "@mantine/core";
 
+import { useSession } from "@homarr/auth/client";
 import type { stringOrTranslation } from "@homarr/translation";
 import { translateIfNecessary } from "@homarr/translation";
 import { useI18n } from "@homarr/translation/client";
@@ -15,14 +17,23 @@ export interface BaseWidgetErrorProps {
 
 export const BaseWidgetError = (props: BaseWidgetErrorProps) => {
   const t = useI18n();
+  const { data: session } = useSession();
+  const [errorTimestamp] = useState(Date.now);
+  const canViewLogs = session?.user.permissions.includes("other-view-logs") ?? false;
 
   return (
     <Stack h="100%" align="center" justify="center" gap="md" data-homarr-widget-error>
       <props.icon size={40} />
       <Stack gap={0}>
         <Text ta="center">{translateIfNecessary(t, props.message)}</Text>
-        {props.showLogsLink && (
-          <Anchor component={Link} href="/manage/tools/logs" target="_blank" ta="center" size="sm">
+        {props.showLogsLink && canViewLogs && (
+          <Anchor
+            component={Link}
+            href={`/manage/tools/logs?focus=${errorTimestamp}`}
+            target="_blank"
+            ta="center"
+            size="sm"
+          >
             {t("common.action.checkLogs")}
           </Anchor>
         )}