From 47724f44eb3a110e5c3ec854e68b1dc54c59082e Mon Sep 17 00:00:00 2001 From: ajnart Date: Wed, 26 Aug 2026 14:47:16 +0200 Subject: [PATCH] fix: harden widget request caching --- .cursor/rules/codebase-context.mdc | 16 +- .../development/widgets-and-integrations.mdx | 33 +- packages/api/src/router/app.ts | 16 +- packages/api/src/router/board.ts | 4 +- .../api/src/router/widgets/indexer-manager.ts | 5 +- packages/common/src/encryption.ts | 12 + .../core/src/infrastructure/redis/client.ts | 5 +- .../integrations/src/base/session-store.ts | 76 +++- packages/redis/src/lib/channel.ts | 330 +++++++++++++++--- packages/redis/src/lib/connection.ts | 6 +- packages/request-handler/src/air-quality.ts | 10 +- .../src/lib/integration-request-handler.ts | 60 +++- .../src/lib/request-handler.ts | 209 +++++++++-- .../request-handler/src/lib/shared-cache.ts | 83 +++-- .../src/lib/widget-request-handler.ts | 12 +- .../src/minecraft-server-status.ts | 10 +- packages/request-handler/src/rss-feeds.ts | 28 +- packages/request-handler/src/stock-price.ts | 16 +- packages/request-handler/src/weather.ts | 13 +- packages/widgets/package.json | 1 + packages/widgets/src/catalog.ts | 2 + packages/widgets/src/manifest.ts | 12 +- .../src/media-transcoding/component.tsx | 15 +- .../widgets/src/media-transcoding/index.ts | 3 +- .../widgets/src/media-transcoding/views.ts | 1 + packages/widgets/src/registry.ts | 69 ++++ 26 files changed, 838 insertions(+), 209 deletions(-) create mode 100644 packages/widgets/src/catalog.ts create mode 100644 packages/widgets/src/media-transcoding/views.ts diff --git a/.cursor/rules/codebase-context.mdc b/.cursor/rules/codebase-context.mdc index 48528cbe9..349673637 100644 --- a/.cursor/rules/codebase-context.mdc +++ b/.cursor/rules/codebase-context.mdc @@ -40,7 +40,7 @@ homarr/ │ ├── db/ # Drizzle schema (3 DB drivers), migrations, queries │ ├── core/ # Env validation, DB/Redis driver factories, logging │ ├── definitions/ # Domain enums: WidgetKind, IntegrationKind, permissions -│ ├── widgets/ # All 39 dashboard widgets (definitions + components) +│ ├── widgets/ # Dashboard widget definitions, components, and lazy registry │ ├── integrations/ # Integration classes (HTTP clients to external apps) │ ├── redis/ # Redis pub/sub channels, caching abstractions │ ├── translation/ # next-intl setup, locale configs, lang JSON files @@ -118,11 +118,15 @@ Selected at runtime via `DB_DRIVER` env (`better-sqlite3` | `mysql2` | `node-pos ### Widget System -Widgets are registered in `packages/widgets/src/index.tsx` as `widgetImports` (must satisfy `Record`). Each widget: -- Has a `definition` created via `createWidgetDefinition(kind, { icon, createOptions, supportedIntegrations? })` -- Uses `optionsBuilder.from(factory => ({ ... }))` for typed options -- Loads dynamically via `next/dynamic` -- Fetches data via tRPC client hooks (backed by cron job caches) +Widget kinds live in `packages/definitions/src/widget.ts`. Integration selection and authorization metadata lives in +`packages/definitions/src/widget-integration-map.ts`. Each widget module exports a typed `definition` and +`componentLoader` created with `createWidgetDefinition(...).withDynamicImport(...)`. + +Register widgets in all three explicit maps in `packages/widgets/src/registry.ts`: the type-only module map, literal +module loaders, and literal component loaders. The two runtime loader maps are deliberately separate so +`packages/widgets/src/manifest.ts` can request a widget's definition chunk and component chunk in parallel while +keeping every lazy import statically discoverable by Next.js. The manifest owns promise caching and clears rejected +entries so later loads can retry. Do not restore an eager root `widgetImports` map. ### Integration System diff --git a/apps/docs/docs/advanced/development/widgets-and-integrations.mdx b/apps/docs/docs/advanced/development/widgets-and-integrations.mdx index 0ee4dd453..6e481bdfa 100644 --- a/apps/docs/docs/advanced/development/widgets-and-integrations.mdx +++ b/apps/docs/docs/advanced/development/widgets-and-integrations.mdx @@ -25,9 +25,11 @@ Keep the UI, options, helpers, and tests together in `packages/widgets/src/ invalidateIntegrationResponseCacheAsync(id))); + } }), delete: permissionRequiredProcedure .requiresPermission("app-full-all") @@ -271,7 +280,12 @@ export const appRouter = createTRPCRouter({ }) .input(byIdSchema) .mutation(async ({ ctx, input }) => { + const linkedIntegrations = await ctx.db.query.integrations.findMany({ + columns: { id: true }, + where: eq(integrations.appId, input.id), + }); await ctx.db.delete(apps).where(eq(apps.id, input.id)); + await Promise.all(linkedIntegrations.map(({ id }) => invalidateIntegrationResponseCacheAsync(id))); }), }); diff --git a/packages/api/src/router/board.ts b/packages/api/src/router/board.ts index ff10c63df..4dec5b2bc 100644 --- a/packages/api/src/router/board.ts +++ b/packages/api/src/router/board.ts @@ -161,8 +161,8 @@ const validateWidgetConfigurationsAsync = async ( ); if (invalidIntegrationIds.length > 0) { throw new TRPCError({ - code: "BAD_REQUEST", - message: `Invalid integration IDs: ${invalidIntegrationIds.join(", ")}`, + code: "NOT_FOUND", + message: "Integration not found", }); } diff --git a/packages/api/src/router/widgets/indexer-manager.ts b/packages/api/src/router/widgets/indexer-manager.ts index e8f7d11e3..7e982e4d9 100644 --- a/packages/api/src/router/widgets/indexer-manager.ts +++ b/packages/api/src/router/widgets/indexer-manager.ts @@ -38,7 +38,7 @@ export const indexerManagerRouter = createTRPCRouter({ testAllIndexers: protectedProcedure .concat(createManyWidgetIntegrationMiddleware("interact", "indexerManager")) .mutation(async ({ ctx }) => { - await Promise.all( + const results = await Promise.allSettled( ctx.integrations.map(async (integration) => { const client = await createIntegrationAsync(integration); await client.testAllAsync().catch((err) => { @@ -51,5 +51,8 @@ export const indexerManagerRouter = createTRPCRouter({ }), ); await indexerManagerRequestHandler.invalidateCacheAsync(ctx.integrations.map(({ id }) => id)); + + const firstFailure = results.find((result) => result.status === "rejected"); + if (firstFailure?.status === "rejected") throw firstFailure.reason; }), }); diff --git a/packages/common/src/encryption.ts b/packages/common/src/encryption.ts index a10451f9f..7b1ed98c3 100644 --- a/packages/common/src/encryption.ts +++ b/packages/common/src/encryption.ts @@ -7,6 +7,7 @@ const algorithm = "aes-256-cbc"; //Using AES encryption // We fallback to a key of 0s if the key was not provided because env validation was skipped // This should only be the case in CI const key = Buffer.from(env.SECRET_ENCRYPTION_KEY || "0".repeat(64), "hex"); +const fingerprintKey = crypto.createHmac("sha256", key).update("homarr-keyed-fingerprint-v1").digest(); export function encryptSecret(text: string): `${string}.${string}` { const initializationVector = crypto.randomBytes(16); @@ -29,3 +30,14 @@ export function decryptSecretWithKey(value: `${string}.${string}`, key: Buffer) decrypted = Buffer.concat([decrypted, decipher.final()]); return decrypted.toString(); } + +export function createKeyedFingerprint(value: string) { + return crypto.createHmac("sha256", fingerprintKey).update(value).digest("hex"); +} + +export function verifyKeyedFingerprint(value: string, fingerprint: string) { + const expected = Buffer.from(createKeyedFingerprint(value), "hex"); + const actual = Buffer.from(fingerprint, "hex"); + if (expected.length !== actual.length) return false; + return crypto.timingSafeEqual(expected, actual); +} diff --git a/packages/core/src/infrastructure/redis/client.ts b/packages/core/src/infrastructure/redis/client.ts index ef68c0474..ffa6de017 100644 --- a/packages/core/src/infrastructure/redis/client.ts +++ b/packages/core/src/infrastructure/redis/client.ts @@ -9,7 +9,7 @@ const defaultRedisOptions = { export type { Redis as RedisClient } from "ioredis"; -export const createRedisClient = () => +export const createRedisClient = (options: RedisOptions = {}) => redisEnv.IS_EXTERNAL ? new Redis({ ...defaultRedisOptions, @@ -23,5 +23,6 @@ export const createRedisClient = () => : undefined, username: redisEnv.USERNAME, password: redisEnv.PASSWORD, + ...options, }) - : new Redis(defaultRedisOptions); + : new Redis({ ...defaultRedisOptions, ...options }); diff --git a/packages/integrations/src/base/session-store.ts b/packages/integrations/src/base/session-store.ts index 0233bb471..8e969dc74 100644 --- a/packages/integrations/src/base/session-store.ts +++ b/packages/integrations/src/base/session-store.ts @@ -1,15 +1,66 @@ import superjson from "superjson"; -import { decryptSecret, encryptSecret } from "@homarr/common/server"; +import { createKeyedFingerprint, decryptSecret, encryptSecret, verifyKeyedFingerprint } from "@homarr/common/server"; import { createLogger } from "@homarr/core/infrastructure/logs"; import { ErrorWithMetadata } from "@homarr/core/infrastructure/logs/error"; import { createGetSetChannel, getIntegrationSessionStoreKey } from "@homarr/redis"; -const logger = createLogger({ module: "sessionStore" }); +import type { IntegrationSecret } from "./types"; -export const createSessionStore = (integration: { id: string }) => { +const logger = createLogger({ module: "sessionStore" }); +const sessionEnvelopeType = "homarr-integration-session"; +const sessionEnvelopeVersion = 1; + +interface SessionStoreIntegration { + id: string; + url: string; + decryptedSecrets: readonly IntegrationSecret[]; +} + +interface SessionEnvelope { + type: typeof sessionEnvelopeType; + version: typeof sessionEnvelopeVersion; + configurationFingerprint: string; + value: TValue; +} + +const compareStrings = (left: string, right: string) => { + if (left < right) return -1; + if (left > right) return 1; + return 0; +}; + +const serializeSessionConfiguration = (integration: SessionStoreIntegration) => { + const secrets = integration.decryptedSecrets + .map(({ kind, value }) => [kind, value] as const) + .toSorted(([leftKind, leftValue], [rightKind, rightValue]) => { + const kindComparison = compareStrings(leftKind, rightKind); + if (kindComparison !== 0) return kindComparison; + return compareStrings(leftValue, rightValue); + }); + + return JSON.stringify({ integrationId: integration.id, url: integration.url, secrets }); +}; + +const isSessionEnvelope = (value: unknown): value is SessionEnvelope => { + if (typeof value !== "object" || value === null) return false; + if (!("value" in value)) return false; + + return ( + "type" in value && + value.type === sessionEnvelopeType && + "version" in value && + value.version === sessionEnvelopeVersion && + "configurationFingerprint" in value && + typeof value.configurationFingerprint === "string" + ); +}; + +export const createSessionStore = (integration: SessionStoreIntegration) => { const channelName = getIntegrationSessionStoreKey(integration.id); const channel = createGetSetChannel<`${string}.${string}`>(channelName); + const serializedConfiguration = serializeSessionConfiguration(integration); + const configurationFingerprint = createKeyedFingerprint(serializedConfiguration); return { async getAsync() { @@ -17,7 +68,16 @@ export const createSessionStore = (integration: { id: string }) => { const value = await channel.getAsync(); if (!value) return null; try { - return superjson.parse(decryptSecret(value)); + const envelope = superjson.parse(decryptSecret(value)); + if (!isSessionEnvelope(envelope)) { + logger.debug("Ignoring legacy or unsupported session", { store: channelName }); + return null; + } + if (!verifyKeyedFingerprint(serializedConfiguration, envelope.configurationFingerprint)) { + logger.debug("Ignoring session for a different integration configuration", { store: channelName }); + return null; + } + return envelope.value; } catch (error) { logger.warn("Failed to load session", { store: channelName, error }); return null; @@ -31,7 +91,13 @@ export const createSessionStore = (integration: { id: string }) => { async setAsync(value: TValue, options?: { ttlSeconds?: number }) { logger.debug("Updating session in store", { store: channelName, ttlSeconds: options?.ttlSeconds }); try { - await channel.setAsync(encryptSecret(superjson.stringify(value)), options); + const envelope: SessionEnvelope = { + type: sessionEnvelopeType, + version: sessionEnvelopeVersion, + configurationFingerprint, + value, + }; + await channel.setAsync(encryptSecret(superjson.stringify(envelope)), options); } catch (error) { logger.error(new ErrorWithMetadata("Failed to save session", { store: channelName }, { cause: error })); } diff --git a/packages/redis/src/lib/channel.ts b/packages/redis/src/lib/channel.ts index 4bff11b19..1cc9d6829 100644 --- a/packages/redis/src/lib/channel.ts +++ b/packages/redis/src/lib/channel.ts @@ -10,6 +10,21 @@ import { createRedisConnection, requireRedisConnection } from "./connection"; const publisher = createRedisConnection(); const lastDataClient = createRedisConnection(); const logger = createLogger({ module: "redisChannel" }); +const boundedCacheClient = createRedisConnection({ + autoResendUnfulfilledCommands: false, + commandTimeout: 500, + enableOfflineQueue: false, + maxRetriesPerRequest: 1, +}); + +interface RedisChannelOptions { + useBoundedCacheClient?: boolean; +} + +const selectRedisClient = (options: RedisChannelOptions) => { + if (options.useBoundedCacheClient) return boundedCacheClient; + return getSetClient; +}; /** * Creates a new pub/sub channel. @@ -100,14 +115,15 @@ export const createListChannel = (name: string) => { * Creates a new redis channel for getting and setting data * @param name name of channel */ -export const createGetSetChannel = (name: string) => { +export const createGetSetChannel = (name: string, options: RedisChannelOptions = {}) => { + const client = selectRedisClient(options); return { /** * Get data from the channel * @returns data or null if not found */ getAsync: async () => { - const data = await requireRedisConnection(getSetClient).get(name); + const data = await requireRedisConnection(client).get(name); return data ? superjson.parse(data) : null; }, /** @@ -118,23 +134,23 @@ export const createGetSetChannel = (name: string) => { setAsync: async (data: TData, options?: { ttlSeconds?: number; ttlMs?: number }) => { if (options?.ttlMs !== undefined) { if (options.ttlMs <= 0) { - await requireRedisConnection(getSetClient).del(name); + await requireRedisConnection(client).del(name); return; } - await requireRedisConnection(getSetClient).set(name, superjson.stringify(data), "PX", options.ttlMs); + await requireRedisConnection(client).set(name, superjson.stringify(data), "PX", options.ttlMs); return; } if (options?.ttlSeconds) { - await requireRedisConnection(getSetClient).set(name, superjson.stringify(data), "EX", options.ttlSeconds); + await requireRedisConnection(client).set(name, superjson.stringify(data), "EX", options.ttlSeconds); return; } - await requireRedisConnection(getSetClient).set(name, superjson.stringify(data)); + await requireRedisConnection(client).set(name, superjson.stringify(data)); }, /** * Remove data from the channel */ removeAsync: async () => { - await requireRedisConnection(getSetClient).del(name); + await requireRedisConnection(client).del(name); }, }; }; @@ -145,18 +161,37 @@ export const createGetSetChannel = (name: string) => { * The token prevents a process from releasing a lock that expired and was * acquired by another process in the meantime. */ -export const createLockChannel = (name: string) => { +export const createLockChannel = (name: string, options: RedisChannelOptions = {}) => { + const selectedClient = selectRedisClient(options); + const releaseIfOwnedAsync = async (token: string) => { + if (!selectedClient) return; + + await selectedClient.eval( + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", + 1, + name, + token, + ); + }; + return { acquireAsync: async (ttlSeconds: number) => { const token = createId(); - const client = getSetClient; + const client = selectedClient; if (!client) return token; - const result = await client.set(name, token, "EX", ttlSeconds, "NX"); - return result === "OK" ? token : null; + try { + const result = await client.set(name, token, "EX", ttlSeconds, "NX"); + return result === "OK" ? token : null; + } catch (error) { + // The SET may have reached Redis even when its reply timed out. Queue a + // token-fenced cleanup so an ambiguous acquisition cannot orphan a lock. + void releaseIfOwnedAsync(token).catch(() => undefined); + throw error; + } }, renewAsync: async (token: string, ttlSeconds: number) => { - const client = getSetClient; + const client = selectedClient; if (!client) return true; const result = await client.eval( @@ -168,21 +203,29 @@ export const createLockChannel = (name: string) => { ); return result === 1; }, - releaseAsync: async (token: string) => { - const client = getSetClient; - if (!client) return; + setIfOwnedAsync: async (token: string, targetName: string, data: TData, ttlMs: number) => { + const client = selectedClient; + if (!client) return true; - await client.eval( - "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end", - 1, + const result = await client.eval( + "if redis.call('get', KEYS[1]) ~= ARGV[1] then return 0 end redis.call('set', KEYS[2], ARGV[2], 'PX', ARGV[3]) return 1", + 2, name, + targetName, token, + superjson.stringify(data), + ttlMs, ); + return result === 1; + }, + releaseAsync: async (token: string) => { + await releaseIfOwnedAsync(token); }, }; }; const integrationCacheGenerationKey = (integrationId: string) => `integration-cache:generation:${integrationId}`; +const integrationCacheGlobalGenerationKey = "integration-cache:global-generation:v1"; export const getIntegrationSessionStoreKey = (integrationId: string) => `session-store:${integrationId}`; interface PendingIntegrationCacheInvalidation { generation: string; @@ -190,10 +233,12 @@ interface PendingIntegrationCacheInvalidation { } const pendingIntegrationCacheInvalidations = new Map(); +let integrationCacheOverflowGeneration: string | undefined; const MAX_PENDING_INTEGRATION_CACHE_INVALIDATIONS = 1000; -const CACHE_GENERATION_REDIS_TIMEOUT_MS = 500; +const CACHE_GENERATION_REDIS_TIMEOUT_MS = 750; const CACHE_GENERATION_SUCCESS_TTL_MS = 1000; const CACHE_GENERATION_UNAVAILABLE_TTL_MS = 5000; +const CACHE_GENERATION_REDIS_TTL_SECONDS = 30 * 24 * 60 * 60; const MAX_CACHE_GENERATION_READS = 1000; export interface IntegrationCacheGeneration { @@ -229,6 +274,29 @@ const withCacheGenerationRedisTimeoutAsync = async (promise: Promise): Pro }); }; +const combineCacheGenerations = ( + globalGeneration: string | null | undefined, + scopedGeneration: string | null | undefined, +) => { + const scopedValue = scopedGeneration ?? "0"; + if (!globalGeneration || globalGeneration === "0") return scopedValue; + return `${globalGeneration}:${scopedValue}`; +}; + +const advanceGlobalCacheGenerationAsync = async (cacheKey: string) => { + const client = boundedCacheClient; + if (!client) return null; + + const results = await withCacheGenerationRedisTimeoutAsync( + client.multi().incr(cacheKey).expire(cacheKey, CACHE_GENERATION_REDIS_TTL_SECONDS).exec(), + ); + const generationResult = results?.[0]; + const expiryResult = results?.[1]; + if (!generationResult || generationResult[0]) throw generationResult?.[0] ?? new Error("Missing generation result"); + if (!expiryResult || expiryResult[0]) throw expiryResult?.[0] ?? new Error("Missing generation expiry result"); + return String(generationResult[1]); +}; + const setCacheGenerationRead = ( cacheKey: string, generation: IntegrationCacheGeneration, @@ -258,6 +326,7 @@ interface GetCachedGenerationOptions { errorMessage: string; errorMetadata: Record; force?: boolean; + throwOnError?: boolean; } const getCachedGenerationAsync = async ({ @@ -267,10 +336,15 @@ const getCachedGenerationAsync = async ({ errorMessage, errorMetadata, force = false, + throwOnError = false, }: GetCachedGenerationOptions): Promise => { if (!force) { const cached = cacheGenerationReads.get(cacheKey); - if (cached && Date.now() < cached.expiresAt) return cached.generation; + if (cached && Date.now() < cached.expiresAt) { + cacheGenerationReads.delete(cacheKey); + cacheGenerationReads.set(cacheKey, cached); + return cached.generation; + } if (cached) cacheGenerationReads.delete(cacheKey); } @@ -285,6 +359,7 @@ const getCachedGenerationAsync = async ({ if (value !== null) generation = { value, isShared: true }; } catch (error) { logCacheGenerationFailure(errorMessage, errorMetadata, error); + if (throwOnError) throw error; } if (cacheGenerationReadsInFlight.get(cacheKey)?.token === token) { @@ -301,28 +376,53 @@ const getCachedGenerationAsync = async ({ }; const advanceIntegrationCacheGenerationAsync = async (integrationId: string) => { - const client = getSetClient; + const client = boundedCacheClient; if (!client) return null; const results = await withCacheGenerationRedisTimeoutAsync( client .multi() .incr(integrationCacheGenerationKey(integrationId)) + .expire(integrationCacheGenerationKey(integrationId), CACHE_GENERATION_REDIS_TTL_SECONDS) .del(getIntegrationSessionStoreKey(integrationId)) + .get(integrationCacheGlobalGenerationKey) .exec(), ); const generationResult = results?.[0]; - const sessionResult = results?.[1]; + const expiryResult = results?.[1]; + const sessionResult = results?.[2]; + const globalGenerationResult = results?.[3]; if (!generationResult || generationResult[0]) throw generationResult?.[0] ?? new Error("Missing generation result"); + if (!expiryResult || expiryResult[0]) throw expiryResult?.[0] ?? new Error("Missing generation expiry result"); if (!sessionResult || sessionResult[0]) throw sessionResult?.[0] ?? new Error("Missing session result"); - return String(generationResult[1]); + if (!globalGenerationResult || globalGenerationResult[0]) { + throw globalGenerationResult?.[0] ?? new Error("Missing global generation result"); + } + return combineCacheGenerations(globalGenerationResult[1] as string | null, String(generationResult[1])); }; const advanceIntegrationResponseCacheGenerationAsync = async (integrationId: string) => { - const client = getSetClient; + const client = boundedCacheClient; if (!client) return null; - return String(await withCacheGenerationRedisTimeoutAsync(client.incr(integrationCacheGenerationKey(integrationId)))); + const cacheKey = integrationCacheGenerationKey(integrationId); + const results = await withCacheGenerationRedisTimeoutAsync( + client + .multi() + .incr(cacheKey) + .expire(cacheKey, CACHE_GENERATION_REDIS_TTL_SECONDS) + .get(integrationCacheGlobalGenerationKey) + .exec(), + ); + const generationResult = results?.[0]; + const expiryResult = results?.[1]; + const globalGenerationResult = results?.[2]; + if (!generationResult || generationResult[0]) throw generationResult?.[0] ?? new Error("Missing generation result"); + if (!expiryResult || expiryResult[0]) throw expiryResult?.[0] ?? new Error("Missing generation expiry result"); + if (!globalGenerationResult || globalGenerationResult[0]) { + throw globalGenerationResult?.[0] ?? new Error("Missing global generation result"); + } + return combineCacheGenerations(globalGenerationResult[1] as string | null, String(generationResult[1])); }; const getIntegrationCacheInvalidationOperation = (clearSession: boolean) => { @@ -340,6 +440,41 @@ const getIntegrationCacheInvalidationOperation = (clearSession: boolean) => { }; }; +let integrationCacheOverflowRecoveryInFlight: Promise | undefined; + +const recoverIntegrationCacheOverflowAsync = async () => { + if (!integrationCacheOverflowGeneration) return true; + if (integrationCacheOverflowRecoveryInFlight) return await integrationCacheOverflowRecoveryInFlight; + + const overflowGeneration = integrationCacheOverflowGeneration; + let promise: Promise; + promise = (async () => { + try { + const generation = await advanceGlobalCacheGenerationAsync(integrationCacheGlobalGenerationKey); + if (generation === null) return false; + + cacheGenerationReads.clear(); + cacheGenerationReadsInFlight.clear(); + if (integrationCacheOverflowGeneration !== overflowGeneration) return false; + integrationCacheOverflowGeneration = undefined; + return true; + } catch (error) { + logCacheGenerationFailure( + "Failed to recover integration cache invalidation overflow", + { operation: "global-generation-advance" }, + error, + ); + return false; + } + })().finally(() => { + if (integrationCacheOverflowRecoveryInFlight === promise) { + integrationCacheOverflowRecoveryInFlight = undefined; + } + }); + integrationCacheOverflowRecoveryInFlight = promise; + return await promise; +}; + export const getIntegrationCacheGenerationAsync = async ( integrationId: string, ): Promise => { @@ -369,12 +504,22 @@ export const getIntegrationCacheGenerationAsync = async ( return generation; } - const client = getSetClient; + if (integrationCacheOverflowGeneration) { + await recoverIntegrationCacheOverflowAsync(); + if (integrationCacheOverflowGeneration) { + return { value: integrationCacheOverflowGeneration, isShared: false }; + } + } + + const client = boundedCacheClient; return await getCachedGenerationAsync({ cacheKey, readAsync: async () => { if (!client) return null; - return (await withCacheGenerationRedisTimeoutAsync(client.get(cacheKey))) ?? "0"; + const [globalGeneration, scopedGeneration] = await withCacheGenerationRedisTimeoutAsync( + client.mget(integrationCacheGlobalGenerationKey, cacheKey), + ); + return combineCacheGenerations(globalGeneration, scopedGeneration); }, fallback: { value: "redis-unavailable", isShared: false }, errorMessage: "Failed to read integration cache generation", @@ -391,26 +536,40 @@ const invalidateIntegrationCacheGenerationAsync = async ( generation: `local-${createId()}`, clearSession: clearSession || previousInvalidation?.clearSession === true, }; - if (pendingIntegrationCacheInvalidations.size >= MAX_PENDING_INTEGRATION_CACHE_INVALIDATIONS) { - const oldestIntegrationId = pendingIntegrationCacheInvalidations.keys().next().value; - if (oldestIntegrationId) pendingIntegrationCacheInvalidations.delete(oldestIntegrationId); + const canTrackInvalidation = + !integrationCacheOverflowGeneration && + (previousInvalidation !== undefined || + pendingIntegrationCacheInvalidations.size < MAX_PENDING_INTEGRATION_CACHE_INVALIDATIONS); + if (canTrackInvalidation) { + pendingIntegrationCacheInvalidations.set(integrationId, pendingInvalidation); + } else if (integrationCacheOverflowGeneration) { + integrationCacheOverflowGeneration = pendingInvalidation.generation; } - pendingIntegrationCacheInvalidations.set(integrationId, pendingInvalidation); const cacheKey = integrationCacheGenerationKey(integrationId); beginCacheGenerationInvalidation(cacheKey, pendingInvalidation.generation); const invalidationOperation = getIntegrationCacheInvalidationOperation(pendingInvalidation.clearSession); - const generation = await getCachedGenerationAsync({ - cacheKey, - readAsync: async () => await invalidationOperation.advanceAsync(integrationId), - fallback: { value: pendingInvalidation.generation, isShared: false }, - errorMessage: invalidationOperation.failureMessage, - errorMetadata: { - integrationId, - operation: invalidationOperation.operation, - }, - force: true, - }); + let generation: IntegrationCacheGeneration; + try { + generation = await getCachedGenerationAsync({ + cacheKey, + readAsync: async () => await invalidationOperation.advanceAsync(integrationId), + fallback: { value: pendingInvalidation.generation, isShared: false }, + errorMessage: invalidationOperation.failureMessage, + errorMetadata: { + integrationId, + operation: invalidationOperation.operation, + }, + force: true, + throwOnError: clearSession, + }); + } catch (error) { + if (!canTrackInvalidation) integrationCacheOverflowGeneration = pendingInvalidation.generation; + throw error; + } + if (!generation.isShared && !canTrackInvalidation) { + integrationCacheOverflowGeneration = pendingInvalidation.generation; + } if ( generation.isShared && pendingIntegrationCacheInvalidations.get(integrationId)?.generation === pendingInvalidation.generation @@ -429,18 +588,72 @@ export const invalidateIntegrationResponseCacheAsync = async (integrationId: str }; const widgetCacheGenerationKey = (namespace: string) => `widget-cache:generation:${namespace}`; +const widgetCacheGlobalGenerationKey = "widget-cache:global-generation:v1"; const pendingWidgetCacheInvalidations = new Map(); +let widgetCacheOverflowGeneration: string | undefined; const MAX_PENDING_WIDGET_CACHE_INVALIDATIONS = 1000; const advanceWidgetCacheGenerationAsync = async (namespace: string) => { - const client = getSetClient; + const client = boundedCacheClient; if (!client) return null; - return String(await withCacheGenerationRedisTimeoutAsync(client.incr(widgetCacheGenerationKey(namespace)))); + const cacheKey = widgetCacheGenerationKey(namespace); + const results = await withCacheGenerationRedisTimeoutAsync( + client + .multi() + .incr(cacheKey) + .expire(cacheKey, CACHE_GENERATION_REDIS_TTL_SECONDS) + .get(widgetCacheGlobalGenerationKey) + .exec(), + ); + const generationResult = results?.[0]; + const expiryResult = results?.[1]; + const globalGenerationResult = results?.[2]; + if (!generationResult || generationResult[0]) throw generationResult?.[0] ?? new Error("Missing generation result"); + if (!expiryResult || expiryResult[0]) throw expiryResult?.[0] ?? new Error("Missing generation expiry result"); + if (!globalGenerationResult || globalGenerationResult[0]) { + throw globalGenerationResult?.[0] ?? new Error("Missing global generation result"); + } + return combineCacheGenerations(globalGenerationResult[1] as string | null, String(generationResult[1])); }; export type WidgetCacheGeneration = IntegrationCacheGeneration; +let widgetCacheOverflowRecoveryInFlight: Promise | undefined; + +const recoverWidgetCacheOverflowAsync = async () => { + if (!widgetCacheOverflowGeneration) return true; + if (widgetCacheOverflowRecoveryInFlight) return await widgetCacheOverflowRecoveryInFlight; + + const overflowGeneration = widgetCacheOverflowGeneration; + let promise: Promise; + promise = (async () => { + try { + const generation = await advanceGlobalCacheGenerationAsync(widgetCacheGlobalGenerationKey); + if (generation === null) return false; + + cacheGenerationReads.clear(); + cacheGenerationReadsInFlight.clear(); + if (widgetCacheOverflowGeneration !== overflowGeneration) return false; + widgetCacheOverflowGeneration = undefined; + return true; + } catch (error) { + logCacheGenerationFailure( + "Failed to recover widget cache invalidation overflow", + { operation: "global-generation-advance" }, + error, + ); + return false; + } + })().finally(() => { + if (widgetCacheOverflowRecoveryInFlight === promise) { + widgetCacheOverflowRecoveryInFlight = undefined; + } + }); + widgetCacheOverflowRecoveryInFlight = promise; + return await promise; +}; + export const getWidgetCacheGenerationAsync = async (namespace: string): Promise => { const cacheKey = widgetCacheGenerationKey(namespace); const pendingGeneration = pendingWidgetCacheInvalidations.get(namespace); @@ -461,12 +674,22 @@ export const getWidgetCacheGenerationAsync = async (namespace: string): Promise< return generation; } - const client = getSetClient; + if (widgetCacheOverflowGeneration) { + await recoverWidgetCacheOverflowAsync(); + if (widgetCacheOverflowGeneration) { + return { value: widgetCacheOverflowGeneration, isShared: false }; + } + } + + const client = boundedCacheClient; return await getCachedGenerationAsync({ cacheKey, readAsync: async () => { if (!client) return null; - return (await withCacheGenerationRedisTimeoutAsync(client.get(cacheKey))) ?? "0"; + const [globalGeneration, scopedGeneration] = await withCacheGenerationRedisTimeoutAsync( + client.mget(widgetCacheGlobalGenerationKey, cacheKey), + ); + return combineCacheGenerations(globalGeneration, scopedGeneration); }, fallback: { value: "redis-unavailable", isShared: false }, errorMessage: "Failed to read widget cache generation", @@ -476,11 +699,15 @@ export const getWidgetCacheGenerationAsync = async (namespace: string): Promise< export const invalidateWidgetCache = (namespace: string): void => { const pendingGeneration = `local-${createId()}`; - if (pendingWidgetCacheInvalidations.size >= MAX_PENDING_WIDGET_CACHE_INVALIDATIONS) { - const oldestNamespace = pendingWidgetCacheInvalidations.keys().next().value; - if (oldestNamespace) pendingWidgetCacheInvalidations.delete(oldestNamespace); + const previousGeneration = pendingWidgetCacheInvalidations.get(namespace); + const canTrackInvalidation = + !widgetCacheOverflowGeneration && + (previousGeneration !== undefined || pendingWidgetCacheInvalidations.size < MAX_PENDING_WIDGET_CACHE_INVALIDATIONS); + if (canTrackInvalidation) { + pendingWidgetCacheInvalidations.set(namespace, pendingGeneration); + } else if (widgetCacheOverflowGeneration) { + widgetCacheOverflowGeneration = pendingGeneration; } - pendingWidgetCacheInvalidations.set(namespace, pendingGeneration); const cacheKey = widgetCacheGenerationKey(namespace); beginCacheGenerationInvalidation(cacheKey, pendingGeneration); @@ -492,6 +719,9 @@ export const invalidateWidgetCache = (namespace: string): void => { errorMetadata: { cacheNamespace: namespace, operation: "generation-advance" }, force: true, }).then((generation) => { + if (!generation.isShared && !canTrackInvalidation) { + widgetCacheOverflowGeneration = pendingGeneration; + } if (generation.isShared && pendingWidgetCacheInvalidations.get(namespace) === pendingGeneration) { pendingWidgetCacheInvalidations.delete(namespace); } diff --git a/packages/redis/src/lib/connection.ts b/packages/redis/src/lib/connection.ts index a97d0572a..a81ab5d6f 100644 --- a/packages/redis/src/lib/connection.ts +++ b/packages/redis/src/lib/connection.ts @@ -1,3 +1,5 @@ +import type { RedisOptions } from "ioredis"; + import type { RedisClient } from "@homarr/core/infrastructure/redis"; import { createRedisClient } from "@homarr/core/infrastructure/redis"; @@ -5,12 +7,12 @@ import { createRedisClient } from "@homarr/core/infrastructure/redis"; * Creates a new Redis connection * @returns redis client */ -export const createRedisConnection = () => { +export const createRedisConnection = (options: RedisOptions = {}) => { if (Boolean(process.env.CI) || Boolean(process.env.DISABLE_REDIS_LOGS)) { return null; } - return createRedisClient(); + return createRedisClient(options); }; export const requireRedisConnection = (client: RedisClient | null): RedisClient => { diff --git a/packages/request-handler/src/air-quality.ts b/packages/request-handler/src/air-quality.ts index 8cccc4248..9e5b62d7d 100644 --- a/packages/request-handler/src/air-quality.ts +++ b/packages/request-handler/src/air-quality.ts @@ -5,7 +5,6 @@ import { z } from "zod"; import { ResponseError } from "@homarr/common/server"; import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http"; -import { withTimeoutAsync } from "@homarr/core/infrastructure/http/timeout"; import { createWidgetRequestHandler } from "./lib/widget-request-handler"; @@ -44,11 +43,12 @@ const requestedVariables = [ ] as const; export const airQualityRequestHandler = createWidgetRequestHandler({ - cacheNamespace: "air-quality", + // Coordinates are a public, continuous key space. Keep this cache process-local and bounded. cacheTtlMs: 15 * 60 * 1000, fallbackToStaleOnError: true, staleIfErrorTtlMs: 24 * 60 * 60 * 1000, - async requestAsync(input: { latitude: number; longitude: number }) { + requestTimeoutMs: 10_000, + async requestAsync(input: { latitude: number; longitude: number }, signal: AbortSignal) { const url = new URL("https://air-quality-api.open-meteo.com/v1/air-quality"); url.searchParams.set("latitude", input.latitude.toString()); url.searchParams.set("longitude", input.longitude.toString()); @@ -57,9 +57,7 @@ export const airQualityRequestHandler = createWidgetRequestHandler({ url.searchParams.set("forecast_days", FORECAST_DAYS.toString()); url.searchParams.set("timezone", "auto"); - const response = await withTimeoutAsync(async (signal) => { - return await fetchWithTrustedCertificatesAsync(url.toString(), { signal }); - }); + const response = await fetchWithTrustedCertificatesAsync(url.toString(), { signal }); if (!response.ok) throw new ResponseError(response); const json: unknown = await response.json(); diff --git a/packages/request-handler/src/lib/integration-request-handler.ts b/packages/request-handler/src/lib/integration-request-handler.ts index 130691e69..b1c6ffd43 100644 --- a/packages/request-handler/src/lib/integration-request-handler.ts +++ b/packages/request-handler/src/lib/integration-request-handler.ts @@ -1,26 +1,63 @@ +import { createKeyedFingerprint } from "@homarr/common/server"; import type { Modify } from "@homarr/common/types"; import type { Integration, IntegrationSecret } from "@homarr/db/schema"; import type { IntegrationKind } from "@homarr/definitions"; import { invalidateIntegrationResponseCacheAsync } from "@homarr/redis"; -import { createIntegrationSharedCacheAsync, hashIntegrationCacheOptions } from "./shared-cache"; +import { createIntegrationSharedCacheAsync, hashSharedCacheInput } from "./shared-cache"; import { createRequestHandler } from "./request-handler"; import type { SharedCacheAdapter } from "./request-handler"; type IntegrationOfKind = Omit & { kind: TKind; - decryptedSecrets: Modify, { value: string }>[]; + decryptedSecrets: Modify< + Pick & Partial>, + { value: string } + >[]; externalUrl: string | null; }; interface Options> { - requestAsync: (integration: IntegrationOfKind, input: TInput) => Promise; + requestAsync: (integration: IntegrationOfKind, input: TInput, signal: AbortSignal) => Promise; cacheTtlMs?: number; fallbackToStaleOnError?: boolean; staleIfErrorTtlMs?: number; + requestTimeoutMs?: number; cacheNamespace?: string; + cacheVersion?: string; } +const getIntegrationCacheIdentity = >({ + integration, + options, +}: { + integration: IntegrationOfKind; + options: TInput; +}) => { + const secretRevisions = integration.decryptedSecrets + .map((secret) => ({ + kind: secret.kind, + fingerprint: createKeyedFingerprint(JSON.stringify([secret.kind, secret.value])), + updatedAt: secret.updatedAt?.toISOString(), + })) + .toSorted((left, right) => { + const kindComparison = left.kind.localeCompare(right.kind); + if (kindComparison !== 0) return kindComparison; + return left.fingerprint.localeCompare(right.fingerprint); + }); + + return hashSharedCacheInput({ + integration: { + id: integration.id, + kind: integration.kind, + url: integration.url, + externalUrl: integration.externalUrl, + secretRevisions, + }, + options, + }); +}; + export const createIntegrationRequestHandler = < TData, TKind extends IntegrationKind, @@ -30,24 +67,29 @@ export const createIntegrationRequestHandler = < ) => { const cacheNamespace = options.cacheNamespace; let getSharedCacheAsync: - | ((input: { integration: IntegrationOfKind; options: TInput }) => Promise>) + | (( + input: { integration: IntegrationOfKind; options: TInput }, + cacheIdentity: string, + ) => Promise>) | undefined; if (cacheNamespace) { - getSharedCacheAsync = async ({ integration, options: itemOptions }) => + getSharedCacheAsync = async ({ integration }, cacheIdentity) => await createIntegrationSharedCacheAsync({ namespace: cacheNamespace, integrationId: integration.id, - cacheOptions: itemOptions, + cacheIdentity, + cacheVersion: options.cacheVersion, }); } const inner = createRequestHandler; options: TInput }>({ - requestAsync: async ({ integration, options: itemOptions }) => options.requestAsync(integration, itemOptions), - getCacheKey: ({ integration, options: itemOptions }) => - `${integration.id}:${hashIntegrationCacheOptions(itemOptions)}`, + requestAsync: async ({ integration, options: itemOptions }, signal) => + options.requestAsync(integration, itemOptions, signal), + getCacheKey: getIntegrationCacheIdentity, cacheTtlMs: options.cacheTtlMs, fallbackToStaleOnError: options.fallbackToStaleOnError, staleIfErrorTtlMs: options.staleIfErrorTtlMs, + requestTimeoutMs: options.requestTimeoutMs, getSharedCacheAsync, }); diff --git a/packages/request-handler/src/lib/request-handler.ts b/packages/request-handler/src/lib/request-handler.ts index f371d257b..1cfc32fa2 100644 --- a/packages/request-handler/src/lib/request-handler.ts +++ b/packages/request-handler/src/lib/request-handler.ts @@ -1,10 +1,11 @@ interface Options> { - requestAsync: (input: TInput) => Promise; + requestAsync: (input: TInput, signal: AbortSignal) => Promise; getCacheKey?: (input: TInput) => string; cacheTtlMs?: number; fallbackToStaleOnError?: boolean; staleIfErrorTtlMs?: number; - getSharedCacheAsync?: (input: TInput) => Promise>; + requestTimeoutMs?: number; + getSharedCacheAsync?: (input: TInput, cacheIdentity: string) => Promise>; } export interface CacheEntry { @@ -18,25 +19,63 @@ export interface SharedCacheAdapter { generation: string; isShared: boolean; getAsync: () => Promise | null | undefined>; - setAsync: (entry: CacheEntry) => Promise; + setAsync: (entry: CacheEntry, refreshLockToken: string) => Promise; acquireRefreshLockAsync: () => Promise; renewRefreshLockAsync: (token: string) => Promise; releaseRefreshLockAsync: (token: string) => Promise; } const MAX_CACHE_SIZE = 1000; +const MAX_INFLIGHT_REQUESTS = 100; +const MAX_UPSTREAM_SETTLEMENT_GRACE_MS = 60_000; const DEFAULT_TTL_MS = 10_000; const DEFAULT_STALE_IF_ERROR_TTL_MS = 5 * 60_000; +const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; const SHARED_REFRESH_POLL_MS = 100; const SHARED_REFRESH_LOCK_RETRY_MS = 1_000; const SHARED_REFRESH_STALE_WAIT_MS = 15_000; +const SHARED_REFRESH_MAX_WAIT_MS = 30_000; // Must remain comfortably below REFRESH_LOCK_TTL_SECONDS in shared-cache.ts. const SHARED_REFRESH_LOCK_RENEW_MS = 5_000; -const delayAsync = async (durationMs: number) => +class RequestHandlerTimeoutError extends Error { + constructor() { + super("Request handler deadline exceeded"); + this.name = RequestHandlerTimeoutError.name; + } +} + +class RequestHandlerOverloadedError extends Error { + constructor() { + super("Request handler concurrency limit exceeded"); + this.name = RequestHandlerOverloadedError.name; + } +} + +const throwIfDeadlineExceeded = (deadlineAt: number) => { + if (Date.now() >= deadlineAt) throw new RequestHandlerTimeoutError(); +}; + +const delayAsync = async (durationMs: number, deadlineAt: number) => { + throwIfDeadlineExceeded(deadlineAt); + const remainingMs = deadlineAt - Date.now(); await new Promise((resolve) => { - setTimeout(resolve, durationMs); + const timer = setTimeout(resolve, Math.min(durationMs, remainingMs)); + timer.unref?.(); }); + throwIfDeadlineExceeded(deadlineAt); +}; + +const waitForSettlementGraceAsync = async (settledAsync: Promise, durationMs: number) => { + await new Promise((resolve) => { + const timer = setTimeout(resolve, durationMs); + timer.unref?.(); + void settledAsync.then(() => { + clearTimeout(timer); + resolve(); + }); + }); +}; const startRefreshLockRenewal = (sharedCache: SharedCacheAdapter, token: string) => { let isOwned = true; @@ -81,8 +120,15 @@ const evictExpired = (cache: Map>) => { export const createRequestHandler = >( options: Options, ) => { + const requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs <= 0) { + throw new Error("Request timeout must be a positive finite duration"); + } + const cache = new Map>(); const inflight = new Map>>(); + const sharedCacheResolutions = new Map>>(); + const upstreamSettlements = new Map>(); let expiryTimer: ReturnType | undefined; let expiryTimerAt: number | undefined; let generation = 0; @@ -90,6 +136,7 @@ export const createRequestHandler = , requestGeneration: number) => { if (generation !== requestGeneration || Date.now() >= entry.staleUntil) return; evictExpired(cache); + cache.delete(key); if (cache.size >= MAX_CACHE_SIZE) { const oldest = cache.keys().next().value; if (oldest !== undefined) cache.delete(oldest); @@ -98,6 +145,64 @@ export const createRequestHandler = { + const entry = cache.get(key); + if (!entry) return undefined; + cache.delete(key); + cache.set(key, entry); + return entry; + }; + + const startRequestWithDeadline = (input: TInput, deadlineAt: number, upstreamKey: string) => { + throwIfDeadlineExceeded(deadlineAt); + if (upstreamSettlements.has(upstreamKey) || upstreamSettlements.size >= MAX_INFLIGHT_REQUESTS) { + throw new RequestHandlerOverloadedError(); + } + + const controller = new AbortController(); + const timeoutError = new RequestHandlerTimeoutError(); + const remainingMs = deadlineAt - Date.now(); + const upstreamRequest = Promise.resolve().then(async () => await options.requestAsync(input, controller.signal)); + const settledAsync = upstreamRequest.then( + () => undefined, + () => undefined, + ); + upstreamSettlements.set(upstreamKey, settledAsync); + void settledAsync.then(() => { + if (upstreamSettlements.get(upstreamKey) === settledAsync) upstreamSettlements.delete(upstreamKey); + }); + + const resultAsync = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + controller.abort(timeoutError); + reject(timeoutError); + }, remainingMs); + timer.unref?.(); + + void upstreamRequest.then(resolve, reject).finally(() => clearTimeout(timer)); + }); + + return { resultAsync, settledAsync }; + }; + + const resolveSharedCacheAsync = (input: TInput, cacheIdentity: string, requestGeneration: number) => { + if (!options.getSharedCacheAsync) return undefined; + + const resolutionKey = `${requestGeneration}:${cacheIdentity}`; + const existing = sharedCacheResolutions.get(resolutionKey); + if (existing) return existing; + if (sharedCacheResolutions.size >= MAX_INFLIGHT_REQUESTS) { + throw new RequestHandlerOverloadedError(); + } + + let promise: Promise>; + promise = options.getSharedCacheAsync(input, cacheIdentity).finally(() => { + if (sharedCacheResolutions.get(resolutionKey) === promise) sharedCacheResolutions.delete(resolutionKey); + }); + sharedCacheResolutions.set(resolutionKey, promise); + return promise; + }; + const scheduleExpiry = (candidateExpiryAt?: number) => { if (candidateExpiryAt !== undefined && expiryTimerAt !== undefined && expiryTimerAt <= candidateExpiryAt) return; if (expiryTimer) clearTimeout(expiryTimer); @@ -123,36 +228,43 @@ export const createRequestHandler = { generation += 1; cache.clear(); - inflight.clear(); if (expiryTimer) clearTimeout(expiryTimer); expiryTimer = undefined; expiryTimerAt = undefined; }, handler: (input: TInput) => ({ async getDataAsync(): Promise<{ data: TData; timestamp: Date }> { + const deadlineAt = Date.now() + requestTimeoutMs; const ttl = options.cacheTtlMs ?? DEFAULT_TTL_MS; const baseKey = options.getCacheKey?.(input) ?? JSON.stringify(input); - const existing = inflight.get(baseKey); + const requestGeneration = generation; + let sharedCache: SharedCacheAdapter | undefined; + if (ttl > 0) { + sharedCache = await resolveSharedCacheAsync(input, baseKey, requestGeneration); + throwIfDeadlineExceeded(deadlineAt); + } + + const key = `${sharedCache?.generation ?? "local"}:${baseKey}`; + const inflightKey = `${requestGeneration}:${key}`; + const existing = inflight.get(inflightKey); if (existing) return existing; - const requestGeneration = generation; + let cached = getFromMemory(key); + const now = Date.now(); + if (cached && now < cached.expiresAt) return cached; + if (cached && now >= cached.staleUntil) { + cache.delete(key); + scheduleExpiry(); + cached = undefined; + } + if (upstreamSettlements.has(inflightKey) || inflight.size >= MAX_INFLIGHT_REQUESTS) { + if (options.fallbackToStaleOnError && cached) return cached; + throw new RequestHandlerOverloadedError(); + } + + let cleanupAfterUpstreamAsync: Promise | undefined; let promise: Promise>; promise = (async () => { - let sharedCache: SharedCacheAdapter | undefined; - if (ttl > 0 && options.getSharedCacheAsync) { - sharedCache = await options.getSharedCacheAsync(input); - } - const key = `${sharedCache?.generation ?? "local"}:${baseKey}`; - - let cached = cache.get(key); - const now = Date.now(); - if (cached && now < cached.expiresAt) return cached; - if (cached && now >= cached.staleUntil) { - cache.delete(key); - scheduleExpiry(); - cached = undefined; - } - let stale = cached; let lockToken: string | null | undefined; @@ -167,9 +279,10 @@ export const createRequestHandler = | undefined; try { - const data = await options.requestAsync(input); + const request = startRequestWithDeadline(input, deadlineAt, inflightKey); + upstreamSettledAsync = request.settledAsync; + const data = await request.resultAsync; const completedAt = Date.now(); const staleIfErrorTtlMs = options.fallbackToStaleOnError ? Math.max(0, options.staleIfErrorTtlMs ?? DEFAULT_STALE_IF_ERROR_TTL_MS) @@ -203,10 +319,15 @@ export const createRequestHandler = 0) { - storeInMemory(key, entry, requestGeneration); - if (sharedCache?.isShared && refreshLockRenewal?.isOwned() !== false) { - await sharedCache.setAsync(entry); + let canStoreInMemory = true; + if (sharedCache?.isShared) { + canStoreInMemory = false; + if (lockToken && refreshLockRenewal?.isOwned() !== false) { + const stored = await sharedCache.setAsync(entry, lockToken); + if (stored !== false || !sharedCache.isShared) canStoreInMemory = true; + } } + if (canStoreInMemory) storeInMemory(key, entry, requestGeneration); } return entry; } catch (error) { @@ -221,14 +342,34 @@ export const createRequestHandler = { + refreshLockRenewal?.stop(); + if (lockToken) await sharedCache?.releaseRefreshLockAsync(lockToken); + }; + if (upstreamSettledAsync) { + const settlementGraceMs = Math.min(requestTimeoutMs, MAX_UPSTREAM_SETTLEMENT_GRACE_MS); + cleanupAfterUpstreamAsync = waitForSettlementGraceAsync(upstreamSettledAsync, settlementGraceMs).then( + cleanupAsync, + ); + } else { + await cleanupAsync(); + } } - })().finally(() => { - if (inflight.get(baseKey) === promise) inflight.delete(baseKey); - }); + })(); - inflight.set(baseKey, promise); + inflight.set(inflightKey, promise); + const removeInflight = () => { + const cleanupAsync = cleanupAfterUpstreamAsync ?? Promise.resolve(); + void cleanupAsync.then( + () => { + if (inflight.get(inflightKey) === promise) inflight.delete(inflightKey); + }, + () => { + if (inflight.get(inflightKey) === promise) inflight.delete(inflightKey); + }, + ); + }; + void promise.then(removeInflight, removeInflight); return promise; }, }), diff --git a/packages/request-handler/src/lib/shared-cache.ts b/packages/request-handler/src/lib/shared-cache.ts index 06df2a89a..6e2c6b199 100644 --- a/packages/request-handler/src/lib/shared-cache.ts +++ b/packages/request-handler/src/lib/shared-cache.ts @@ -15,9 +15,14 @@ import type { IntegrationCacheGeneration, WidgetCacheGeneration } from "@homarr/ import type { CacheEntry, SharedCacheAdapter } from "./request-handler"; const logger = createLogger({ module: "sharedRequestCache" }); -const REDIS_OPERATION_TIMEOUT_MS = 250; +// The dedicated cache client rejects commands after 500 ms. This guard stays +// above that deadline so lock outcomes are never abandoned while still pending. +const REDIS_OPERATION_TIMEOUT_MS = 750; const REFRESH_LOCK_TTL_SECONDS = 15; +const SHARED_CACHE_ENVELOPE_VERSION = 2; +const DEFAULT_HANDLER_CACHE_VERSION = "v1"; const namespacePattern = /^[a-z0-9][a-z0-9:-]*$/; +const cacheVersionPattern = /^[a-z0-9][a-z0-9._-]*$/; const isRecord = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); @@ -47,27 +52,40 @@ const withTimeoutAsync = async (promise: Promise, durationMs: number): Pro }); }; -const isCacheEntry = (value: unknown): value is CacheEntry => { - if (!isRecord(value)) return false; +interface SharedCacheEnvelope { + version: typeof SHARED_CACHE_ENVELOPE_VERSION; + entry: CacheEntry; +} + +const isSharedCacheEnvelope = (value: unknown): value is SharedCacheEnvelope => { + if (!isRecord(value) || value.version !== SHARED_CACHE_ENVELOPE_VERSION || !isRecord(value.entry)) return false; + + const entry = value.entry; + if (!Object.hasOwn(entry, "data") || !(entry.timestamp instanceof Date)) return false; + const timestamp = entry.timestamp.getTime(); return ( - value.timestamp instanceof Date && - typeof value.expiresAt === "number" && - Number.isFinite(value.expiresAt) && - typeof value.staleUntil === "number" && - Number.isFinite(value.staleUntil) + Number.isFinite(timestamp) && + typeof entry.expiresAt === "number" && + Number.isFinite(entry.expiresAt) && + typeof entry.staleUntil === "number" && + Number.isFinite(entry.staleUntil) && + timestamp <= entry.expiresAt && + entry.expiresAt <= entry.staleUntil ); }; interface IntegrationSharedCacheOptions { namespace: string; integrationId: string; - cacheOptions: Record; + cacheIdentity: string; + cacheVersion?: string; } interface RedisSharedCacheOptions { namespace: string; cacheKind: "integration" | "widget"; cacheIdentity: string; + cacheVersion: string; generation: string; metadata?: Record; isShared?: boolean; @@ -90,15 +108,16 @@ const createRedisSharedCache = ({ namespace, cacheKind, cacheIdentity, + cacheVersion, generation, metadata = {}, isShared: initialIsShared = true, }: RedisSharedCacheOptions): SharedCacheAdapter => { const logMetadata = { cacheNamespace: namespace, cacheKind, ...metadata }; - const payloadChannel = createGetSetChannel>( - `${cacheKind}-cache:payload:v1:${namespace}:${generation}:${cacheIdentity}`, - ); - const lock = createLockChannel(`${cacheKind}-cache:lock:v1:${namespace}:${generation}:${cacheIdentity}`); + const payloadName = `${cacheKind}-cache:payload:v2:${cacheVersion}:${namespace}:${generation}:${cacheIdentity}`; + const lockName = `${cacheKind}-cache:lock:v2:${cacheVersion}:${namespace}:${generation}:${cacheIdentity}`; + const payloadChannel = createGetSetChannel(payloadName, { useBoundedCacheClient: true }); + const lock = createLockChannel(lockName, { useBoundedCacheClient: true }); let isShared = initialIsShared; const runRedisOperationAsync = async (operation: string, callback: () => Promise, fallback: T): Promise => { @@ -120,7 +139,7 @@ const createRedisSharedCache = ({ getAsync: async () => { const value = await runRedisOperationAsync("payload-read", () => payloadChannel.getAsync(), undefined); if (value === undefined || value === null) return value; - if (isCacheEntry(value)) return value; + if (isSharedCacheEnvelope(value)) return value.entry as CacheEntry; logger.warn("Ignored invalid Redis response cache entry", { ...logMetadata, @@ -129,12 +148,16 @@ const createRedisSharedCache = ({ await runRedisOperationAsync("payload-remove", () => payloadChannel.removeAsync(), undefined); return null; }, - setAsync: async (entry) => { + setAsync: async (entry, refreshLockToken) => { const retentionMs = entry.staleUntil - Date.now(); - if (retentionMs <= 0) return; - await runRedisOperationAsync( + if (retentionMs <= 0) return false; + const envelope: SharedCacheEnvelope = { + version: SHARED_CACHE_ENVELOPE_VERSION, + entry, + }; + return await runRedisOperationAsync( "payload-write", - () => payloadChannel.setAsync(entry, { ttlMs: retentionMs }), + () => lock.setIfOwnedAsync(refreshLockToken, payloadName, envelope, retentionMs), undefined, ); }, @@ -151,22 +174,29 @@ const createRedisSharedCache = ({ undefined, ), releaseRefreshLockAsync: async (token) => { - await runRedisOperationAsync("refresh-lock-release", () => lock.releaseAsync(token), undefined); + try { + await withTimeoutAsync(lock.releaseAsync(token), REDIS_OPERATION_TIMEOUT_MS); + } catch (error) { + logRedisFailure("refresh-lock-release", logMetadata, error); + } }, }; }; interface WidgetSharedCacheOptions { namespace: string; - cacheInput: Record; + cacheIdentity: string; + cacheVersion?: string; } export const createWidgetSharedCacheAsync = async ({ namespace, - cacheInput, + cacheIdentity, + cacheVersion = DEFAULT_HANDLER_CACHE_VERSION, }: WidgetSharedCacheOptions): Promise> => { if (typeof window !== "undefined") throw new Error("Widget request caching is server-only"); if (!namespacePattern.test(namespace)) throw new Error(`Invalid widget cache namespace '${namespace}'`); + if (!cacheVersionPattern.test(cacheVersion)) throw new Error(`Invalid widget cache version '${cacheVersion}'`); const metadata = { cacheNamespace: namespace }; let generation: WidgetCacheGeneration; @@ -180,7 +210,8 @@ export const createWidgetSharedCacheAsync = async ({ return createRedisSharedCache({ namespace, cacheKind: "widget", - cacheIdentity: hashSharedCacheInput(cacheInput), + cacheIdentity, + cacheVersion, generation: generation.value, isShared: generation.isShared, }); @@ -189,10 +220,12 @@ export const createWidgetSharedCacheAsync = async ({ export const createIntegrationSharedCacheAsync = async ({ namespace, integrationId, - cacheOptions, + cacheIdentity, + cacheVersion = DEFAULT_HANDLER_CACHE_VERSION, }: IntegrationSharedCacheOptions): Promise> => { if (typeof window !== "undefined") throw new Error("Integration request caching is server-only"); if (!namespacePattern.test(namespace)) throw new Error(`Invalid integration cache namespace '${namespace}'`); + if (!cacheVersionPattern.test(cacheVersion)) throw new Error(`Invalid integration cache version '${cacheVersion}'`); const metadata = { cacheNamespace: namespace, integrationId }; let generation: IntegrationCacheGeneration; @@ -203,11 +236,11 @@ export const createIntegrationSharedCacheAsync = async ({ generation = { value: "redis-unavailable", isShared: false }; } - const optionsHash = hashIntegrationCacheOptions(cacheOptions); const sharedCache = createRedisSharedCache({ namespace, cacheKind: "integration", - cacheIdentity: `${integrationId}:${optionsHash}`, + cacheIdentity, + cacheVersion, generation: generation.value, metadata: { integrationId }, isShared: generation.isShared, diff --git a/packages/request-handler/src/lib/widget-request-handler.ts b/packages/request-handler/src/lib/widget-request-handler.ts index 09edc0b20..19882500d 100644 --- a/packages/request-handler/src/lib/widget-request-handler.ts +++ b/packages/request-handler/src/lib/widget-request-handler.ts @@ -5,23 +5,26 @@ import { createRequestHandler } from "./request-handler"; import type { SharedCacheAdapter } from "./request-handler"; interface Options> { - requestAsync: (input: TInput) => Promise; + requestAsync: (input: TInput, signal: AbortSignal) => Promise; cacheTtlMs?: number; fallbackToStaleOnError?: boolean; staleIfErrorTtlMs?: number; + requestTimeoutMs?: number; cacheNamespace?: string; + cacheVersion?: string; } export const createWidgetRequestHandler = >( options: Options, ) => { const cacheNamespace = options.cacheNamespace; - let getSharedCacheAsync: ((input: TInput) => Promise>) | undefined; + let getSharedCacheAsync: ((input: TInput, cacheIdentity: string) => Promise>) | undefined; if (cacheNamespace) { - getSharedCacheAsync = async (input) => + getSharedCacheAsync = async (_input, cacheIdentity) => await createWidgetSharedCacheAsync({ namespace: cacheNamespace, - cacheInput: input, + cacheIdentity, + cacheVersion: options.cacheVersion, }); } @@ -30,6 +33,7 @@ export const createWidgetRequestHandler = - fetchWithTrustedCertificatesAsync(`https://api.mcsrvstat.us${path}`, { signal }), - ); + const response = await fetchWithTrustedCertificatesAsync(`https://api.mcsrvstat.us${path}`, { signal }); return responseSchema.parse(await response.json()); }, }); diff --git a/packages/request-handler/src/rss-feeds.ts b/packages/request-handler/src/rss-feeds.ts index c1dbce8c2..0ffcfd381 100644 --- a/packages/request-handler/src/rss-feeds.ts +++ b/packages/request-handler/src/rss-feeds.ts @@ -10,19 +10,23 @@ import { createWidgetRequestHandler } from "./lib/widget-request-handler"; const logger = createLogger({ module: "rssFeedsRequestHandler" }); export const rssFeedsRequestHandler = createWidgetRequestHandler({ - cacheNamespace: "rss-feeds", - async requestAsync(input: { url: string; count: number }) { - const result = (await extract(input.url, { - getExtraEntryFields: (feedEntry) => { - const media = attemptGetImageFromEntry(input.url, feedEntry); - if (!media) { - return {}; - } - return { - enclosure: media, - }; + // Authorized editors can supply arbitrary URLs, so keep this cache process-local and bounded. + async requestAsync(input: { url: string; count: number }, signal) { + const result = (await extract( + input.url, + { + getExtraEntryFields: (feedEntry) => { + const media = attemptGetImageFromEntry(input.url, feedEntry); + if (!media) { + return {}; + } + return { + enclosure: media, + }; + }, }, - })) as ExtendedFeedData; + { signal }, + )) as ExtendedFeedData; return { ...result, diff --git a/packages/request-handler/src/stock-price.ts b/packages/request-handler/src/stock-price.ts index 60d466956..4ad6324f0 100644 --- a/packages/request-handler/src/stock-price.ts +++ b/packages/request-handler/src/stock-price.ts @@ -1,19 +1,17 @@ import { z } from "zod/v4"; import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http"; -import { withTimeoutAsync } from "@homarr/core/infrastructure/http/timeout"; import { createWidgetRequestHandler } from "./lib/widget-request-handler"; export const fetchStockPriceHandler = createWidgetRequestHandler({ - cacheNamespace: "stock-price", - async requestAsync(input: { stock: string; timeRange: string; timeInterval: string }) { - const response = await withTimeoutAsync(async (signal) => { - return await fetchWithTrustedCertificatesAsync( - `https://query1.finance.yahoo.com/v8/finance/chart/${input.stock}?range=${input.timeRange}&interval=${input.timeInterval}`, - { signal }, - ); - }); + // Symbols are supplied through a public procedure and form an unbounded key space. + requestTimeoutMs: 10_000, + async requestAsync(input: { stock: string; timeRange: string; timeInterval: string }, signal) { + const response = await fetchWithTrustedCertificatesAsync( + `https://query1.finance.yahoo.com/v8/finance/chart/${input.stock}?range=${input.timeRange}&interval=${input.timeInterval}`, + { signal }, + ); const data = dataSchema.parse(await response.json()); if ("error" in data) { diff --git a/packages/request-handler/src/weather.ts b/packages/request-handler/src/weather.ts index b56f0bebd..a6b694529 100644 --- a/packages/request-handler/src/weather.ts +++ b/packages/request-handler/src/weather.ts @@ -5,7 +5,6 @@ import { z } from "zod"; import { ResponseError } from "@homarr/common/server"; import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http"; -import { withTimeoutAsync } from "@homarr/core/infrastructure/http/timeout"; import { createWidgetRequestHandler } from "./lib/widget-request-handler"; @@ -15,7 +14,10 @@ dayjs.extend(timezone); const HOURLY_FORECAST_LENGTH = 24; const DAILY_FORECAST_LENGTH = 7; -const requestWeatherAsync = async (input: { latitude: number; longitude: number }): Promise => { +const requestWeatherAsync = async ( + input: { latitude: number; longitude: number }, + signal: AbortSignal, +): Promise => { const url = new URL("https://api.open-meteo.com/v1/forecast"); url.searchParams.set("latitude", input.latitude.toString()); url.searchParams.set("longitude", input.longitude.toString()); @@ -68,9 +70,7 @@ const requestWeatherAsync = async (input: { latitude: number; longitude: number url.searchParams.set("forecast_hours", HOURLY_FORECAST_LENGTH.toString()); url.searchParams.set("forecast_days", DAILY_FORECAST_LENGTH.toString()); - const res = await withTimeoutAsync(async (signal) => { - return await fetchWithTrustedCertificatesAsync(url.toString(), { signal }); - }); + const res = await fetchWithTrustedCertificatesAsync(url.toString(), { signal }); if (!res.ok) throw new ResponseError(res); const json: unknown = await res.json(); @@ -140,8 +140,9 @@ const requestWeatherAsync = async (input: { latitude: number; longitude: number }; export const weatherRequestHandler = createWidgetRequestHandler({ - cacheNamespace: "weather:at-location", + // Coordinates are a public, continuous key space. Keep this cache process-local and bounded. cacheTtlMs: 5 * 60 * 1000, + requestTimeoutMs: 10_000, requestAsync: requestWeatherAsync, }); diff --git a/packages/widgets/package.json b/packages/widgets/package.json index 7836f5a35..2daffc1b5 100644 --- a/packages/widgets/package.json +++ b/packages/widgets/package.json @@ -20,6 +20,7 @@ "./custom-api/custom-jsx-display": "./src/custom-api/custom-jsx-display.tsx", "./custom-api/extract-display-data": "./src/custom-api/extract-display-data.ts", "./assistant/context": "./src/assistant/context.tsx", + "./catalog": "./src/catalog.ts", "./definition": "./src/definition.ts", "./errors": "./src/errors/component.tsx", "./manifest": "./src/manifest.ts", diff --git a/packages/widgets/src/catalog.ts b/packages/widgets/src/catalog.ts new file mode 100644 index 000000000..2d00c1b90 --- /dev/null +++ b/packages/widgets/src/catalog.ts @@ -0,0 +1,2 @@ +/** @deprecated Import widgetCatalogIcons from "@homarr/ui/widget-icons" instead. */ +export { widgetCatalogIcons } from "@homarr/ui/widget-icons"; diff --git a/packages/widgets/src/manifest.ts b/packages/widgets/src/manifest.ts index 954971ce3..f7e3f44a4 100644 --- a/packages/widgets/src/manifest.ts +++ b/packages/widgets/src/manifest.ts @@ -8,7 +8,7 @@ import type { SettingsContextProps } from "@homarr/settings/creator"; import type { WidgetComponentProps, WidgetDefinition } from "./definition"; import type { WidgetOptionDefinition } from "./options"; import type { WidgetImports } from "./registry"; -import { widgetModuleLoaders } from "./registry"; +import { widgetComponentLoaders, widgetModuleLoaders } from "./registry"; type RegisteredWidgetModule = WidgetImports[WidgetKind]; type RegisteredWidgetComponentModule = Awaited>; @@ -87,15 +87,7 @@ export const loadWidgetModule = (kind: TKind) => correlateWidgetPromise(kind, "module", loadRegisteredWidgetModule(kind)); const loadRegisteredWidgetComponent = createRetryableLoader( - new Map Promise>( - widgetKinds.map((kind) => [ - kind, - async () => { - const widgetModule = await loadRegisteredWidgetModule(kind); - return widgetModule.componentLoader(); - }, - ]), - ), + new Map Promise>(objectEntries(widgetComponentLoaders)), ); export const loadWidgetComponent = (kind: TKind) => diff --git a/packages/widgets/src/media-transcoding/component.tsx b/packages/widgets/src/media-transcoding/component.tsx index d85db6a58..adc06fb10 100644 --- a/packages/widgets/src/media-transcoding/component.tsx +++ b/packages/widgets/src/media-transcoding/component.tsx @@ -3,25 +3,13 @@ import { useEffect, useMemo, useState } from "react"; import type { ReactNode } from "react"; import { getQueryKey } from "@trpc/react-query"; -import { - Button, - Divider, - Group, - Pagination, - Paper, - ScrollArea, - Select, - SimpleGrid, - Stack, - Text, -} from "@mantine/core"; +import { Button, Divider, Group, Pagination, Paper, ScrollArea, Select, SimpleGrid, Stack, Text } from "@mantine/core"; import { IconClipboardList, IconCpu2, IconReportAnalytics } from "@tabler/icons-react"; import { clientApi } from "@homarr/api/client"; import { useI18n } from "@homarr/translation/client"; import type { TablerIcon } from "@homarr/ui"; -import { views } from "."; import { WidgetEmptyState } from "../common/empty-state"; import type { WidgetComponentProps } from "../definition"; import { getUsableWidgetQueryData } from "../common/query-state"; @@ -30,6 +18,7 @@ import { HealthCheckStatus } from "./health-check-status"; import { QueuePanel } from "./panels/queue.panel"; import { StatisticsPanel } from "./panels/statistics.panel"; import { WorkersPanel } from "./panels/workers.panel"; +import { views } from "./views"; type View = (typeof views)[number]; diff --git a/packages/widgets/src/media-transcoding/index.ts b/packages/widgets/src/media-transcoding/index.ts index 4e9961b31..ea13dbe52 100644 --- a/packages/widgets/src/media-transcoding/index.ts +++ b/packages/widgets/src/media-transcoding/index.ts @@ -6,8 +6,9 @@ import { getWidgetIntegrationConfig } from "@homarr/definitions"; import { createWidgetDefinition, matchesWidgetRuntimeQuery } from "../definition"; import { optionsBuilder } from "../options"; +import { views } from "./views"; -export const views = ["workers", "queue", "statistics"] as const; +export { views }; export const { componentLoader, definition } = createWidgetDefinition("mediaTranscoding", { icon: IconTransform, diff --git a/packages/widgets/src/media-transcoding/views.ts b/packages/widgets/src/media-transcoding/views.ts new file mode 100644 index 000000000..38f3c6167 --- /dev/null +++ b/packages/widgets/src/media-transcoding/views.ts @@ -0,0 +1 @@ +export const views = ["workers", "queue", "statistics"] as const; diff --git a/packages/widgets/src/registry.ts b/packages/widgets/src/registry.ts index 8a9a6e3d0..747b0eddc 100644 --- a/packages/widgets/src/registry.ts +++ b/packages/widgets/src/registry.ts @@ -186,6 +186,75 @@ export type WidgetImports = { wud: typeof wud; }; +type WidgetComponentLoaders = { + [TKind in WidgetKind]: WidgetImports[TKind]["componentLoader"]; +}; + +// Keep component imports literal and separate from widgetModuleLoaders. The +// manifest starts both chunk requests together on a cold widget load; resolving +// this loader through the widget module would serialize those requests. +export const widgetComponentLoaders = { + clock: () => import("./clock/component"), + airQuality: () => import("./air-quality/component"), + countdown: () => import("./countdown/component"), + timer: () => import("./timer/component"), + app: () => import("./app/component"), + assistant: () => import("./assistant/component"), + archiveTeamWarrior: () => import("./archive-team-warrior/component"), + anchorNote: () => import("./anchor-note/component"), + notebook: () => import("./notebook/component"), + iframe: () => import("./iframe/component"), + video: () => import("./video/component"), + dnsHoleSummary: () => import("./dns-hole/summary/component"), + dnsHoleControls: () => import("./dns-hole/controls/component"), + "smartHome-entityState": () => import("./smart-home/entity-state/component"), + "smartHome-executeAutomation": () => import("./smart-home/execute-automation/component"), + stockPrice: () => import("./stocks/component"), + mediaServer: () => import("./media-server/component"), + calendar: () => import("./calendar/component"), + downloads: () => import("./downloads/component"), + "mediaRequests-requestList": () => import("./media-requests/list/component"), + "mediaRequests-requestStats": () => import("./media-requests/stats/component"), + mediaMissing: () => import("./media-missing/component"), + networkControllerSummary: () => import("./network-controller/summary/component"), + networkControllerStatus: () => import("./network-controller/network-status/component"), + rssFeed: () => import("./rssFeed/component"), + bookmarks: () => import("./bookmarks/component"), + bazarr: () => import("./bazarr/component"), + indexerManager: () => import("./indexer-manager/component"), + healthMonitoring: () => import("./health-monitoring/component"), + mediaTranscoding: () => import("./media-transcoding/component"), + minecraftServerStatus: () => import("./minecraft/server-status/component"), + dockerContainers: () => import("./docker/component"), + releases: () => import("./releases/component"), + firewall: () => import("./firewall/component"), + notifications: () => import("./notifications/component"), + mediaReleases: () => import("./media-releases/component"), + systemResources: () => import("./system-resources/component"), + coolify: () => import("./coolify/component"), + systemDisks: () => import("./system-disks/component"), + timetable: () => import("./timetable/component"), + "immich-serverStats": () => import("./immich/server-stats/component"), + "immich-albumCarousel": () => import("./immich/album-carousel/component"), + paperlessNgx: () => import("./paperless-ngx/component"), + patchmon: () => import("./patchmon/component"), + tracearr: () => import("./tracearr/component"), + speedtestTracker: () => import("./speedtest-tracker/component"), + uptimeKuma: () => import("./uptime-kuma/component"), + audioStats: () => import("./audio-stats/component"), + umami: () => import("./umami/component"), + vpn: () => import("./vpn/component"), + ups: () => import("./ups/component"), + beszelSystemTable: () => import("./beszel-system-table/component"), + beszelSystemGrid: () => import("./beszel-system-grid/component"), + beszelAlerts: () => import("./beszel-alerts/component"), + beszelSystemStats: () => import("./beszel-system-stats/component"), + traefik: () => import("./traefik/component"), + customApi: () => import("./custom-api/component"), + weather: () => import("./weather/component"), + wud: () => import("./wud/component"), +} satisfies WidgetComponentLoaders; + export type WidgetImportKey = keyof WidgetImports; export type inferSupportedIntegrations = (WidgetImports[TKind]["definition"] extends {