From 4ebaea29fd4b68917c4137909994900fefdda8a5 Mon Sep 17 00:00:00 2001 From: ajnart Date: Tue, 11 Aug 2026 19:47:11 +0200 Subject: [PATCH] Harden Homarr provider integration --- .../advanced/environment-variables/index.mdx | 25 +-- .../custom-widgets/agent-authoring.mdx | 17 +- apps/docs/docs/management/homarr-provider.mdx | 14 +- apps/docs/docs/management/workshop/index.mdx | 17 +- .../assistant-provider-activity.tsx | 17 +- .../widgets/assistant-provider-widget.tsx | 3 +- .../chat/assistant-provider-options.ts | 6 +- .../components/assistant/assistant-panel.tsx | 35 ++-- .../assistant-provider-quota.spec.ts | 9 +- .../assistant/assistant-provider-quota.ts | 11 ++ .../assistant/assistant-provider.tsx | 61 ++++--- apps/workshop/homarr_provider.go | 163 ++++++++++-------- apps/workshop/tests/workshop.integration.mjs | 47 ++++- packages/api/src/router/assistant.ts | 2 +- .../api/src/router/test/assistant.spec.ts | 67 +++++++ .../scripts/ai-assistant-evaluation.ts | 14 +- .../custom-widgets/scripts/ai-evaluation.ts | 30 +++- .../scripts/evaluate-ai-authoring.ts | 23 ++- .../src/test/ai-evaluation.spec.ts | 20 +++ packages/db/schema/mysql.ts | 1 + packages/db/schema/postgresql.ts | 1 + packages/db/schema/sqlite.ts | 1 + packages/translation/src/lang/en.json | 2 + 23 files changed, 427 insertions(+), 159 deletions(-) diff --git a/apps/docs/docs/advanced/environment-variables/index.mdx b/apps/docs/docs/advanced/environment-variables/index.mdx index 86444e541..d28d1f6fe 100644 --- a/apps/docs/docs/advanced/environment-variables/index.mdx +++ b/apps/docs/docs/advanced/environment-variables/index.mdx @@ -25,17 +25,19 @@ This is especially useful for [docker secrets](https://docs.docker.com/engine/sw Using the `PUID` and `PGID` will require you to set the correct permissions on the mounted volumes and if used the docker socket. See more in the [Running as a different user](/docs/advanced/running-as-different-user) documentation. ::: -| Environment Variable | Description | Possible values | Default | -| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | ----------------------------- | -| `PUID` | User ID to run the container as | Any valid user ID | `0` | -| `PGID` | Group ID to run the container as | Any valid group ID | `0` | -| `LOG_LEVEL` | Log level to use | `debug` / `info` / `warn` / `error` | `info` | -| `NO_EXTERNAL_CONNECTION` | Disables Homarr's internet-backed requests, including update checks, weather, and RSS feeds | `true` or `false` | `false` | -| `ENABLE_DNS_CACHING` | Enables dns caching. Enabled by default, but if you experience issues with IPv6 or static IPs, see [#4006](https://github.com/homarr-labs/homarr/issues/4006) or disable this flag | `true` or `false` | `true` | -| `HOMARR_DISABLE_IPV6` | Disables the IPv6 listener of the internal nginx proxy, even when the host supports IPv6. IPv6 is normally disabled automatically when the host has no IPv6 configured. See [#4596](https://github.com/homarr-labs/homarr/issues/4596) | `true` or `false` | `false` | -| `HOMARR_WEBSITE_URL` | Main Homarr documentation/site base used by runtime links. Must be an absolute HTTP(S) URL. | HTTP(S) URL | `https://homarr.dev` | -| `WORKSHOP_API_URL` | PocketBase API used by Homarr and standalone documentation. | HTTP(S) URL | `HOMARR_WEBSITE_URL` | -| `WORKSHOP_WEB_URL` | Public Workshop page opened from Homarr. | HTTP(S) URL | `HOMARR_WEBSITE_URL/workshop` | +| Environment Variable | Description | Possible values | Default | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | ------------------------------ | +| `PUID` | User ID to run the container as | Any valid user ID | `0` | +| `PGID` | Group ID to run the container as | Any valid group ID | `0` | +| `LOG_LEVEL` | Log level to use | `debug` / `info` / `warn` / `error` | `info` | +| `NO_EXTERNAL_CONNECTION` | Disables Homarr's internet-backed requests, including update checks, weather, and RSS feeds | `true` or `false` | `false` | +| `ENABLE_DNS_CACHING` | Enables dns caching. Enabled by default, but if you experience issues with IPv6 or static IPs, see [#4006](https://github.com/homarr-labs/homarr/issues/4006) or disable this flag | `true` or `false` | `true` | +| `HOMARR_DISABLE_IPV6` | Disables the IPv6 listener of the internal nginx proxy, even when the host supports IPv6. IPv6 is normally disabled automatically when the host has no IPv6 configured. See [#4596](https://github.com/homarr-labs/homarr/issues/4596) | `true` or `false` | `false` | +| `HOMARR_WEBSITE_URL` | Main Homarr documentation/site base used by runtime links. Must be an absolute HTTP(S) URL. | HTTP(S) URL | `https://homarr.dev` | +| `WORKSHOP_API_URL` | PocketBase API used by Homarr and standalone documentation. | HTTP(S) URL | `HOMARR_WEBSITE_URL` | +| `WORKSHOP_WEB_URL` | Public Workshop page opened from Homarr. | HTTP(S) URL | `HOMARR_WEBSITE_URL/workshop` | +| `HOMARR_AI_DAILY_REQUEST_LIMIT` | Initial Homarr provider request allowance stored for each Workshop user. Existing per-user limits are retained. | Integer from `1` to `100000` | `50` | +| `HOMARR_AI_OPENROUTER_BASE_URL` | OpenRouter-compatible upstream used by the Workshop Homarr provider proxy. | HTTP(S) URL | `https://openrouter.ai/api/v1` | ## Authentication @@ -114,6 +116,7 @@ The advanced deployments environment variables should only be used if you know w | `DB_MIGRATIONS_DISABLED` | Disable db migrations. For example for helm charts. | | `WORKSHOP_PUBLIC_ORIGIN` | Origin used in PocketBase-generated links and email. It must be set explicitly in production and must match the externally reachable Workshop origin. | | `PB_ALLOWED_ORIGINS` | Browser origins accepted by PocketBase. Use `*` for the central public community so arbitrary self-hosted Homarr instances can use the API and OAuth; private staging may use a comma-separated origin allowlist. | +| `OPENROUTER_API_KEY` | Server-only OpenRouter key used by the Workshop Homarr provider proxy. The provider is unavailable when empty. | All public URL variables reject non-HTTP protocols, embedded credentials, query strings, and fragments, and normalize trailing slashes when Homarr or Docusaurus starts. The Docusaurus-only `WORKSHOP_URL` name is accepted as a deprecated alias for diff --git a/apps/docs/docs/management/custom-widgets/agent-authoring.mdx b/apps/docs/docs/management/custom-widgets/agent-authoring.mdx index 4e41bba03..0adc7181d 100644 --- a/apps/docs/docs/management/custom-widgets/agent-authoring.mdx +++ b/apps/docs/docs/management/custom-widgets/agent-authoring.mdx @@ -125,6 +125,18 @@ To exercise the real Homarr Assistant lifecycle instead of accepting a returned OPENROUTER_API_KEY=... pnpm --filter @homarr/custom-widgets test:assistant-ai ``` +The same suites can target the Homarr provider with a Workshop PocketBase token instead of an OpenRouter key: + +```sh +AI_PROVIDER_BASE_URL=https://homarr.dev/api/ai/v1 \ +AI_PROVIDER_API_KEY= \ +AI_PROVIDER_MODEL=homarr/deepseek-v4-flash-latest \ +pnpm --filter @homarr/custom-widgets test:assistant-ai +``` + +`AI_PROVIDER_JUDGE_MODEL` can select a separate judge when the configured endpoint offers one. The Homarr provider +offers exactly one model, so generation and review both use `homarr/deepseek-v4-flash-latest`. + The assistant test exposes the same lazy skill, live schema, component catalog, targeted component references, validation, preview-query, and create-from-preview tools as the product. It refuses to save a definition that was not validated exactly or whose preview queries were not all tested. Multi-request fixtures return a different payload for @@ -137,8 +149,9 @@ tool traces, validated manifests, raw reviews, and detailed category scores unde they consume a configured provider account and exercise a nondeterministic model. The repository's ordinary tests cover the lifecycle enforcement, fixture routing, score calculation, and failure behavior without network access. -Each scenario stops as soon as it passes and has a hard limit of ten generation attempts. Override the models or -per-scenario limit with `OPENROUTER_GENERATOR_MODEL`, `OPENROUTER_JUDGE_MODEL`, and +Each scenario stops as soon as it passes and has a hard limit of ten generation attempts. Override the endpoint, +models, or per-scenario limit with `AI_PROVIDER_BASE_URL`, `AI_PROVIDER_API_KEY`, `AI_PROVIDER_MODEL`, +`AI_PROVIDER_JUDGE_MODEL`, and `CUSTOM_WIDGET_AI_MAX_LOOPS`. Generated artifacts are local and ignored by Git; promote a reviewed, validated result into the bundled-widget source deliberately. diff --git a/apps/docs/docs/management/homarr-provider.mdx b/apps/docs/docs/management/homarr-provider.mdx index 8fbf8791e..a06635187 100644 --- a/apps/docs/docs/management/homarr-provider.mdx +++ b/apps/docs/docs/management/homarr-provider.mdx @@ -57,12 +57,16 @@ share the prompt, API source, and screenshots so other Homarr users can improve The open-source proxy is part of the Workshop image. Operators configure it with server-only environment variables: -| Variable | Purpose | Default | -| --- | --- | --- | -| `OPENROUTER_API_KEY` | Upstream key used only by the Workshop proxy | Disabled when empty | -| `HOMARR_AI_DAILY_REQUEST_LIMIT` | Initial daily limit stored for each new user | `50` | -| `HOMARR_AI_OPENROUTER_BASE_URL` | OpenRouter-compatible upstream base URL | `https://openrouter.ai/api/v1` | +| Variable | Purpose | Default | +| ------------------------------- | -------------------------------------------- | ------------------------------ | +| `OPENROUTER_API_KEY` | Upstream key used only by the Workshop proxy | Disabled when empty | +| `HOMARR_AI_DAILY_REQUEST_LIMIT` | Initial daily limit stored for each new user | `50` | +| `HOMARR_AI_OPENROUTER_BASE_URL` | OpenRouter-compatible upstream base URL | `https://openrouter.ai/api/v1` | Changing the global default affects new quota records. Existing per-user limits remain unchanged so administrators can support future account plans. Run `pnpm test:workshop` to verify authentication, streaming, UTC reset, atomic quota enforcement, tool-call accounting, private records, anonymous public activity, realtime delivery, and migration rollback. + +The Custom Widget AI evaluation suites can use this proxy by setting `AI_PROVIDER_BASE_URL` to this service's +`/api/ai/v1` endpoint and `AI_PROVIDER_API_KEY` to a disposable Workshop user token. See +[Agent authoring](./custom-widgets/agent-authoring#evaluate-the-copied-prompt) for the complete command. diff --git a/apps/docs/docs/management/workshop/index.mdx b/apps/docs/docs/management/workshop/index.mdx index 97c26c40a..16aeebc17 100644 --- a/apps/docs/docs/management/workshop/index.mdx +++ b/apps/docs/docs/management/workshop/index.mdx @@ -111,13 +111,16 @@ SMTP for comment, report, and removal notices. The public URL contract is: -| Variable | Purpose | Default | -| ------------------------ | ------------------------------------------------- | ----------------------------- | -| `HOMARR_WEBSITE_URL` | Documentation/site base used by Homarr links | `https://homarr.dev` | -| `WORKSHOP_API_URL` | PocketBase API used by Homarr and standalone docs | `HOMARR_WEBSITE_URL` | -| `WORKSHOP_WEB_URL` | Public Workshop page linked from Homarr | `HOMARR_WEBSITE_URL/workshop` | -| `WORKSHOP_PUBLIC_ORIGIN` | PocketBase-generated links and email | Required in production | -| `PB_ALLOWED_ORIGINS` | PocketBase browser origins | `*` for the central community | +| Variable | Purpose | Default | +| ------------------------------- | ------------------------------------------------- | ------------------------------ | +| `HOMARR_WEBSITE_URL` | Documentation/site base used by Homarr links | `https://homarr.dev` | +| `WORKSHOP_API_URL` | PocketBase API used by Homarr and standalone docs | `HOMARR_WEBSITE_URL` | +| `WORKSHOP_WEB_URL` | Public Workshop page linked from Homarr | `HOMARR_WEBSITE_URL/workshop` | +| `WORKSHOP_PUBLIC_ORIGIN` | PocketBase-generated links and email | Required in production | +| `PB_ALLOWED_ORIGINS` | PocketBase browser origins | `*` for the central community | +| `OPENROUTER_API_KEY` | Server-only key for the Homarr provider proxy | Provider disabled when empty | +| `HOMARR_AI_DAILY_REQUEST_LIMIT` | Initial per-user daily provider allowance | `50` | +| `HOMARR_AI_OPENROUTER_BASE_URL` | OpenRouter-compatible proxy upstream | `https://openrouter.ai/api/v1` | The deprecated docs-only `WORKSHOP_URL` alias remains for one release; `WORKSHOP_API_URL` takes precedence. The combined Workshop image writes these public URLs into its browser runtime configuration whenever the container starts; diff --git a/apps/docs/src/components/assistant-provider/assistant-provider-activity.tsx b/apps/docs/src/components/assistant-provider/assistant-provider-activity.tsx index 3f01e4805..980ed9821 100644 --- a/apps/docs/src/components/assistant-provider/assistant-provider-activity.tsx +++ b/apps/docs/src/components/assistant-provider/assistant-provider-activity.tsx @@ -23,9 +23,11 @@ const statusLabel = { failed: "Failed", }; +const activityDate = (value: string) => new Date(value.includes("T") ? value : value.replace(" ", "T")); + const mergeActivity = (current: WorkshopAssistantActivity[], next: WorkshopAssistantActivity) => [next, ...current.filter((item) => item.id !== next.id)] - .toSorted((left, right) => Date.parse(right.created) - Date.parse(left.created)) + .toSorted((left, right) => activityDate(right.created).getTime() - activityDate(left.created).getTime()) .slice(0, 10); export const AssistantProviderActivity = ({ compact = false }: { compact?: boolean }) => { @@ -44,13 +46,18 @@ export const AssistantProviderActivity = ({ compact = false }: { compact?: boole try { const initial = await backend.listAssistantActivity(10); if (active) setActivities(initial); - unsubscribe = await backend.pocketBase.collection("assistant_activity").subscribe("*", (event) => { + const nextUnsubscribe = await backend.pocketBase.collection("assistant_activity").subscribe("*", (event) => { const parsed = workshopAssistantActivitySchema.safeParse(event.record); if (!active || !parsed.success) return; setConnected(true); setActivities((current) => mergeActivity(current, parsed.data)); }); - if (active) setConnected(true); + if (active) { + unsubscribe = nextUnsubscribe; + setConnected(true); + } else { + nextUnsubscribe(); + } } catch { if (active) setConnected(false); } finally { @@ -93,8 +100,8 @@ export const AssistantProviderActivity = ({ compact = false }: { compact?: boole return ( -