refactor: simplify widget and integration architecture
This commit is contained in:
1 parent
c988c35b29
commit
8f9efd0074
179 files changed
+1509
-4187
No files matched your search
@@ -8,206 +8,58 @@ tags:
|
||||
sidebar_position: 2
|
||||
---
|
||||
|
||||
Homarr is moving native widgets and integrations to vertical feature modules: the stable identity, lightweight metadata,
|
||||
client entry, server registration, cache policy, and documentation source live beside the implementation. Generated
|
||||
files preserve static imports and package boundaries for Next.js, tRPC, and Docusaurus.
|
||||
Homarr keeps feature registration explicit and typed. There is no generator or generated registry to run after a
|
||||
change. A new feature is available when its typed metadata and lazy loader are registered.
|
||||
|
||||
Registration means availability. There is no separate feature-enable list or runtime kill switch.
|
||||
## Add a widget
|
||||
|
||||
## Current status
|
||||
Keep the UI, options, helpers, and tests together in `packages/widgets/src/<widget>/`. The usual folder only needs an
|
||||
`index.ts` definition and a lazily imported `component.tsx`; complex widgets can add local files as needed.
|
||||
|
||||
Weather and Downloads are the reference widget modules. Their folders now own registration and canonical docs:
|
||||
1. Add the stable kind to `packages/definitions/src/widget.ts` and its documentation slug to the direct typed map in
|
||||
`packages/definitions/src/docs/widget-doc-slugs.ts`.
|
||||
2. Export `definition` and `componentLoader` from `packages/widgets/src/<widget>/index.ts` with
|
||||
`createWidgetDefinition`. The local definition owns its icon, query keys, refresh interval, options, matchers, and
|
||||
errors. Keep the component behind `withDynamicImport` so adding a widget does not load it on every dashboard.
|
||||
3. For an integration-backed widget, add its integration kinds and selection rules once to
|
||||
`packages/definitions/src/widget-integration-map.ts`, then spread `getWidgetIntegrationConfig(kind)` into the local
|
||||
definition. This server-safe config is also the API authorization boundary, avoiding a package cycle without
|
||||
duplicating widget metadata.
|
||||
4. Register the module in the explicit lazy-loader and type-only maps in `packages/widgets/src/registry.ts`, then add
|
||||
its icon to `packages/ui/src/widget-icons.ts`. The two registry entries preserve exact option/component inference;
|
||||
keep them in the same order so drift is obvious in review.
|
||||
5. If the widget needs server data, add its tRPC router under `packages/api/src/router/widgets/` and add its explicit
|
||||
lazy import to `packages/api/src/router/widgets/index.ts`. Keep upstream requests and Redis response caching in
|
||||
`@homarr/request-handler`. Use the widget integration middleware for integration-backed procedures.
|
||||
6. Add canonical documentation in `apps/docs/docs/widgets/<slug>/`.
|
||||
|
||||
```text
|
||||
packages/widgets/src/weather/
|
||||
├── module.ts # authored manifest and client/server data policy
|
||||
├── index.ts # options and presentation-specific definition
|
||||
├── component.tsx # client UI
|
||||
├── module.generated.ts # generated icon projection; do not edit
|
||||
└── docs/ # canonical MDX, metadata, and assets
|
||||
```
|
||||
Weather is the small integration-free example. Downloads shows integration selection, polling, server caching,
|
||||
mutations, partial upstream failures, and a larger options surface.
|
||||
|
||||
The remaining widgets still use the compatibility registries and can be migrated one folder at a time without changing
|
||||
their persisted `WidgetKind` or tRPC paths. The generated and legacy entries are merged behind the same runtime loaders.
|
||||
## Add an integration
|
||||
|
||||
Beszel is the reference integration module. Its folder owns the manifest, implementation, capability helpers, tests,
|
||||
and canonical docs. The generator projects its metadata into definitions and its constructor into the lazy integration
|
||||
factory without eagerly importing the implementation.
|
||||
Keep the HTTP client and its capability-specific types in `packages/integrations/src/<integration>/`.
|
||||
|
||||
The server router implementation is not physically inside these first two widget folders yet. `@homarr/widgets`
|
||||
currently imports `@homarr/api/client`; making `@homarr/api` import widget server files would create a package cycle and
|
||||
risk shipping server code to the browser. The module already owns the lazy router registration, and the neutral server
|
||||
operation types are the cycle-breaking seam for the next migration stage.
|
||||
1. Add the typed definition to `packages/definitions/src/integration.ts`, including credential alternatives,
|
||||
categories, documentation slug, ports, Docker aliases, and onboarding metadata when relevant.
|
||||
2. Add an explicit lazy creator to `packages/integrations/src/base/creator.ts`. The literal import keeps the
|
||||
implementation out of the initial server bundle and makes the registration easy to find.
|
||||
3. Implement only the interfaces required by the widgets that use the service.
|
||||
4. Add canonical documentation in `apps/docs/docs/integrations/<slug>/`. Its `index.ts` must satisfy
|
||||
`IntegrationDefinition` so invalid metadata fails type checking.
|
||||
|
||||
## The module manifest
|
||||
Beszel is a compact example with several widgets sharing one integration client.
|
||||
|
||||
`module.ts` is pure and server-safe. It must not import React, tRPC, an integration implementation, secrets, or a request
|
||||
handler. It describes import locations as strings so the generator can emit literal imports into the correct package.
|
||||
## Data and errors
|
||||
|
||||
```ts
|
||||
import { defineWidgetModule } from "@homarr/definitions";
|
||||
- TanStack Query owns browser caching and refresh behavior. Redis owns shared server response caching.
|
||||
- Cache raw service responses, never authorization decisions or decrypted credentials. Integration-backed cache keys
|
||||
include the integration identity and are invalidated after mutations.
|
||||
- Keep validation and authorization on the server. Client-side checks improve UX but do not replace tRPC guards.
|
||||
- Use the shared logger with safe identifiers, operation names, duration, and the original error cause. Never log
|
||||
secrets or full upstream payloads.
|
||||
- Multi-integration reads should return successful providers when one service fails and surface a clear error when all
|
||||
providers fail.
|
||||
|
||||
export default defineWidgetModule({
|
||||
kind: "example",
|
||||
icon: "IconSparkles",
|
||||
clientEntry: ".",
|
||||
documentation: {
|
||||
slug: "example",
|
||||
sourceDirectory: "docs",
|
||||
},
|
||||
integration: {
|
||||
categories: ["downloadClient"],
|
||||
serverMaxIntegrations: 1,
|
||||
},
|
||||
operations: [
|
||||
{
|
||||
name: "getData",
|
||||
path: ["widget", "example", "getData"],
|
||||
client: {
|
||||
refetchIntervalSeconds: 30,
|
||||
persist: true,
|
||||
},
|
||||
serverCache: {
|
||||
namespace: "example:data",
|
||||
ttlMs: 30_000,
|
||||
scope: "integration",
|
||||
},
|
||||
},
|
||||
],
|
||||
routers: [
|
||||
{
|
||||
namespace: "example",
|
||||
module: "./example",
|
||||
exportName: "exampleRouter",
|
||||
},
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
Run the synchronizer after adding or changing a module:
|
||||
|
||||
```sh
|
||||
pnpm widgets:new example --icon IconSparkles
|
||||
pnpm widgets:sync
|
||||
pnpm widgets:sync:check
|
||||
```
|
||||
|
||||
`widgets:new` creates a minimal client-only folder and immediately runs the synchronizer. Replace the empty component,
|
||||
add options, and extend the manifest with operations or router registrations as needed; it never edits a hand-written
|
||||
central feature list.
|
||||
|
||||
The check command is read-only and reports drift. The synchronizer rejects duplicate widget kinds, docs slugs, router
|
||||
namespaces with conflicting targets, operation names, and cache namespaces. It also verifies client entries, router
|
||||
modules, and local docs before producing anything.
|
||||
|
||||
It generates these compatibility projections:
|
||||
|
||||
| Consumer | Generated projection |
|
||||
| ---------------------- | -------------------------------------------------------------------- |
|
||||
| Definitions and policy | `packages/definitions/src/generated/widget-modules.ts` |
|
||||
| Client lazy loading | `packages/widgets/src/generated/widget-modules.ts` |
|
||||
| Lightweight icons | `packages/ui/src/generated/widget-icons.ts` |
|
||||
| Lazy tRPC routers | `packages/api/src/router/widgets/generated-registry.ts` |
|
||||
| Docusaurus | `apps/docs/docs/widgets/<slug>/` from the module's canonical `docs/` |
|
||||
|
||||
Generated client and server registries contain explicit import literals. Do not replace them with a variable dynamic
|
||||
import: explicit paths let Next.js and Turbopack discover chunks without eagerly loading every widget.
|
||||
|
||||
Use `pnpm features:sync` or `pnpm features:sync:check` when a change spans widgets and integrations.
|
||||
|
||||
## Client and server data caching
|
||||
|
||||
The operation is the shared identity for both cache layers:
|
||||
|
||||
- The tRPC path drives the TanStack Query key.
|
||||
- `refetchIntervalSeconds` configures the root Query Client without importing widget UI code.
|
||||
- `persist` controls session-scoped browser persistence. Persisted payloads are size-bounded, sanitized, and separated
|
||||
by the authenticated user and permission scope.
|
||||
- `serverCache` is imported by the request handler through `getWidgetServerCachePolicy`, so its namespace and TTL are
|
||||
not copied into another file.
|
||||
|
||||
Server reads use bounded in-process L1 caching and a shared Redis L2. Inputs are canonically serialized and SHA-256
|
||||
hashed; raw URLs, options, and secrets never appear in Redis keys or cache logs. Redis operations and lock waits are
|
||||
bounded, refreshes use token-safe distributed locks, and stale data can be retained for the declared error window.
|
||||
Redis failure degrades to local caching and produces structured logs with safe metadata and the original cause.
|
||||
|
||||
Integration-backed cache keys include the integration ID and its Redis generation. Mutations advance that generation,
|
||||
which invalidates all replicas without scanning keys. Integration-free widget handlers use a namespace generation;
|
||||
manual refreshes such as Docker inventory invalidation therefore clear both local and shared values. A zero TTL keeps
|
||||
only overlapping in-flight request deduplication and never stores a Redis payload, which is appropriate for realtime
|
||||
or subscription data.
|
||||
|
||||
Generation reads are process-cached for one second and concurrent misses are deduplicated, so a hot L1 hit does not add
|
||||
a Redis round trip. Redis failure backs off for five seconds instead of producing a request storm; local invalidations
|
||||
take effect immediately and other replicas converge within the one-second generation window.
|
||||
|
||||
TanStack Query remains the browser cache and Redis remains the server cache. A browser persistence hit never bypasses
|
||||
server authorization, and Redis never caches permission decisions or decrypted credentials.
|
||||
|
||||
## Server operations and tRPC
|
||||
|
||||
The neutral contract includes helpers for queries, mutations, subscriptions, and routes:
|
||||
|
||||
- `defineWidgetServerOperation`
|
||||
- `defineWidgetServerSubscription`
|
||||
- `defineWidgetServerRoute`
|
||||
- `defineWidgetServerModule`
|
||||
|
||||
They describe validated input, authorization intent, cache policy, an `AbortSignal`, and a framework-neutral resolver.
|
||||
The future API adapter will inject tRPC procedure builders, integration middleware, logging, and error normalization.
|
||||
This dependency direction lets a local `server.ts` be consumed by API code without the widget client importing API
|
||||
runtime code.
|
||||
|
||||
Until that adapter replaces the current client dependency, keep actual upstream calls in `@homarr/request-handler` and
|
||||
the tRPC compatibility router in `packages/api/src/router/widgets/`. Put the router registration in the local manifest;
|
||||
do not add it to the hand-written registry.
|
||||
|
||||
An integration-backed procedure must use the widget-aware middleware. It derives compatible kinds and maximum
|
||||
selection limits from the module/catalog, resolves only the requested records, decrypts secrets on the server, and
|
||||
checks direct user and group permissions before the procedure runs. Use `query` permission for reads and `interact`
|
||||
permission for mutations.
|
||||
|
||||
## Errors and logs
|
||||
|
||||
- Root tRPC instrumentation records the procedure path, operation type, user, duration, and outcome.
|
||||
- Integration middleware records safe requested/resolved IDs and kinds plus denial reasons; it never logs credentials.
|
||||
- Integration calls are timed and normalized into public errors while preserving a structured server-side cause.
|
||||
- Multi-integration reads use `settleIntegrationQueries` so one failed service does not erase successful providers.
|
||||
- Widget components read through the shared usable-data helper: stale data remains visible after a refresh error, while
|
||||
first-load failures reach the common widget error boundary and logs link.
|
||||
- Lazy client, router, and integration imports cache successful and in-flight loads, evict failed promises, and log
|
||||
load duration so a transient chunk failure can retry.
|
||||
|
||||
## Complex modules
|
||||
|
||||
The manifest is the common path, not a limit on implementation size.
|
||||
|
||||
- Beszel can group multiple stable widget kinds behind one router and keep live statistics on an SSE subscription with
|
||||
Redis payload caching disabled.
|
||||
- Assistant can use a route/stream adapter for chat while keeping client runtime state and tools in subfolders.
|
||||
- A proxy-only widget can expose a server route without inventing a polling query.
|
||||
- A widget can add custom query matchers, context actions, prefetching, or nested client components while retaining the
|
||||
same registration, permissions, logging, error, and cache contracts.
|
||||
|
||||
## Integration modules
|
||||
|
||||
Create an integration module with an explicit icon URL:
|
||||
|
||||
```sh
|
||||
pnpm integrations:new example --icon-url https://example.com/example.svg
|
||||
pnpm integrations:sync:check
|
||||
```
|
||||
|
||||
The scaffold creates a compileable connection test, pure `module.ts`, and canonical docs in
|
||||
`packages/integrations/src/example/`. Add capability interfaces and service methods beside that implementation, then
|
||||
connect them to widget request handlers. Declare real credential alternatives, categories, Docker discovery aliases,
|
||||
ports, and onboarding metadata in the local module instead of editing the central definitions and creator registries.
|
||||
|
||||
The integration synchronizer validates kinds, credential kinds, categories, ports, URLs, documentation, creator
|
||||
exports, and collisions. It generates the definitions projection, a typed lazy creator registry with literal imports,
|
||||
and Docusaurus copies whose edit links point back to the canonical module docs.
|
||||
|
||||
Beszel is migrated to this path. Other integrations still use the compatibility registry and can move independently
|
||||
without changing persisted `IntegrationKind` values or widget capability checks. Keep integrations server-only,
|
||||
implement only their capability interfaces, and use narrow schema/interface exports instead of the broad runtime
|
||||
barrel while migrating them.
|
||||
Registration files intentionally contain boring, explicit imports. This small amount of repetition preserves exact
|
||||
TypeScript inference, gives Next.js discoverable lazy chunks, and keeps Ctrl+click navigation reliable.
|
||||
@@ -1 +0,0 @@
|
||||
packages/integrations/src/beszel/docs
|
||||
@@ -1,7 +1,9 @@
|
||||
import type { IntegrationDefinition } from "@site/src/types";
|
||||
|
||||
export const beszelIntegration = {
|
||||
name: "Beszel",
|
||||
description:
|
||||
"Beszel is a lightweight server monitoring platform with Docker stats, historical data, and alert functions.",
|
||||
iconUrl: "https://cdn.jsdelivr.net/gh/homarr-labs/dashboard-icons@master/svg/beszel.svg",
|
||||
path: "../../integrations/beszel",
|
||||
} as const;
|
||||
} satisfies IntegrationDefinition;
|
||||
@@ -1 +0,0 @@
|
||||
packages/widgets/src/downloads/docs
|
||||
@@ -1 +0,0 @@
|
||||
packages/widgets/src/weather/docs
|
||||
@@ -2,37 +2,8 @@ import type * as Preset from "@docusaurus/preset-classic";
|
||||
import type { Config } from "@docusaurus/types";
|
||||
import { themes as prismThemes } from "prism-react-renderer";
|
||||
import { resolveHomarrUrlConfig } from "@homarr/workshop/schema";
|
||||
import { generatedIntegrationDocumentationSources, generatedWidgetDocumentationSources } from "@homarr/definitions";
|
||||
const a11yEmoji = require("@fec/remark-a11y-emoji");
|
||||
|
||||
const githubDocsEditRoot = "https://github.com/homarr-labs/homarr/edit/dev";
|
||||
|
||||
const getDocumentationEditUrl = (docPath: string) => {
|
||||
const integrationPath = /^integrations\/([^/]+)\/(.+)$/u.exec(docPath);
|
||||
const integrationSlug = integrationPath?.[1];
|
||||
const integrationRelativePath = integrationPath?.[2];
|
||||
if (
|
||||
integrationSlug &&
|
||||
integrationRelativePath &&
|
||||
Object.hasOwn(generatedIntegrationDocumentationSources, integrationSlug)
|
||||
) {
|
||||
const source =
|
||||
generatedIntegrationDocumentationSources[
|
||||
integrationSlug as keyof typeof generatedIntegrationDocumentationSources
|
||||
];
|
||||
return `${githubDocsEditRoot}/${source}/${integrationRelativePath}`;
|
||||
}
|
||||
|
||||
const widgetPath = /^widgets\/([^/]+)\/(.+)$/u.exec(docPath);
|
||||
const slug = widgetPath?.[1];
|
||||
const relativePath = widgetPath?.[2];
|
||||
if (slug && relativePath && Object.hasOwn(generatedWidgetDocumentationSources, slug)) {
|
||||
const source = generatedWidgetDocumentationSources[slug as keyof typeof generatedWidgetDocumentationSources];
|
||||
return `${githubDocsEditRoot}/${source}/${relativePath}`;
|
||||
}
|
||||
return `${githubDocsEditRoot}/apps/docs/docs/${docPath}`;
|
||||
};
|
||||
|
||||
const deprecatedWorkshopUrl = process.env.WORKSHOP_URL;
|
||||
if (deprecatedWorkshopUrl && !process.env.WORKSHOP_API_URL) {
|
||||
console.warn("WORKSHOP_URL is deprecated; use WORKSHOP_API_URL instead.");
|
||||
@@ -94,7 +65,7 @@ const config: Config = {
|
||||
{
|
||||
docs: {
|
||||
sidebarPath: require.resolve("./sidebars.js"),
|
||||
editUrl: ({ docPath }) => getDocumentationEditUrl(docPath),
|
||||
editUrl: ({ docPath }) => `https://github.com/homarr-labs/homarr/edit/dev/apps/docs/docs/${docPath}`,
|
||||
remarkPlugins: [a11yEmoji],
|
||||
exclude: [],
|
||||
showLastUpdateAuthor: false,
|
||||
|
||||
@@ -2,13 +2,17 @@ import type { LoadContext, Plugin } from "@docusaurus/types";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import { integrationDocSlugs } from "@homarr/definitions/docs/integration-doc-slugs";
|
||||
import { widgetDocSlugs } from "@homarr/definitions/docs/widget-doc-slugs";
|
||||
import { getIntegrationDocumentationSlug, integrationKinds } from "@homarr/definitions/integration";
|
||||
|
||||
const slugMaps = {
|
||||
Integration: { slugs: integrationDocSlugs, folder: "integrations" },
|
||||
Widget: { slugs: widgetDocSlugs, folder: "widgets" },
|
||||
} as const;
|
||||
const documentationEntries = [
|
||||
{
|
||||
label: "Integration",
|
||||
folder: "integrations",
|
||||
entries: integrationKinds.map((kind) => [kind, getIntegrationDocumentationSlug(kind)] as const),
|
||||
},
|
||||
{ label: "Widget", folder: "widgets", entries: Object.entries(widgetDocSlugs) },
|
||||
] as const;
|
||||
|
||||
export default function validateDocsCoveragePlugin(context: LoadContext): Plugin {
|
||||
return {
|
||||
@@ -17,8 +21,8 @@ export default function validateDocsCoveragePlugin(context: LoadContext): Plugin
|
||||
const docsDir = path.resolve(context.siteDir, "docs");
|
||||
const missing: string[] = [];
|
||||
|
||||
for (const [label, { slugs, folder }] of Object.entries(slugMaps)) {
|
||||
for (const [kind, slug] of Object.entries(slugs)) {
|
||||
for (const { label, folder, entries } of documentationEntries) {
|
||||
for (const [kind, slug] of entries) {
|
||||
if (!slug) continue;
|
||||
if (!fs.existsSync(path.join(docsDir, folder, slug))) {
|
||||
missing.push(`${label} "${kind}" -> docs/${folder}/${slug}/`);
|
||||
|
||||
Reference in new issue
Block a user