diff --git a/apps/docs/docs/advanced/development/widgets-and-integrations.mdx b/apps/docs/docs/advanced/development/widgets-and-integrations.mdx index 955dcd81a..03111897c 100644 --- a/apps/docs/docs/advanced/development/widgets-and-integrations.mdx +++ b/apps/docs/docs/advanced/development/widgets-and-integrations.mdx @@ -30,6 +30,10 @@ packages/widgets/src/weather/ The remaining widgets still use the compatibility registries and can be migrated one folder at a time without changing their persisted `WidgetKind` or tRPC paths. The generated and legacy entries are merged behind the same runtime loaders. +Beszel is the reference integration module. Its folder owns the manifest, implementation, capability helpers, tests, +and canonical docs. The generator projects its metadata into definitions and its constructor into the lazy integration +factory without eagerly importing the implementation. + The server router implementation is not physically inside these first two widget folders yet. `@homarr/widgets` currently imports `@homarr/api/client`; making `@homarr/api` import widget server files would create a package cycle and risk shipping server code to the browser. The module already owns the lazy router registration, and the neutral server @@ -109,6 +113,8 @@ It generates these compatibility projections: Generated client and server registries contain explicit import literals. Do not replace them with a variable dynamic import: explicit paths let Next.js and Turbopack discover chunks without eagerly loading every widget. +Use `pnpm features:sync` or `pnpm features:sync:check` when a change spans widgets and integrations. + ## Client and server data caching The operation is the shared identity for both cache layers: @@ -131,6 +137,10 @@ manual refreshes such as Docker inventory invalidation therefore clear both loca only overlapping in-flight request deduplication and never stores a Redis payload, which is appropriate for realtime or subscription data. +Generation reads are process-cached for one second and concurrent misses are deduplicated, so a hot L1 hit does not add +a Redis round trip. Redis failure backs off for five seconds instead of producing a request storm; local invalidations +take effect immediately and other replicas converge within the one-second generation window. + TanStack Query remains the browser cache and Redis remains the server cache. A browser persistence hit never bypasses server authorization, and Redis never caches permission decisions or decrypted credentials. @@ -179,10 +189,25 @@ The manifest is the common path, not a limit on implementation size. - A widget can add custom query matchers, context actions, prefetching, or nested client components while retaining the same registration, permissions, logging, error, and cache contracts. -## Legacy integration path +## Integration modules -Integration metadata is currently derived from `integrationDefs`, while implementation constructors are loaded lazily -through `@homarr/integrations/factory`. Keep integrations server-only, implement only their capability interfaces, and -use narrow schema/interface exports instead of the broad runtime barrel. The same local-manifest and generated-registry -shape is being introduced for integrations so metadata, constructor registration, and canonical docs can move beside -each implementation without changing `IntegrationKind` or widget capability checks. +Create an integration module with an explicit icon URL: + +```sh +pnpm integrations:new example --icon-url https://example.com/example.svg +pnpm integrations:sync:check +``` + +The scaffold creates a compileable connection test, pure `module.ts`, and canonical docs in +`packages/integrations/src/example/`. Add capability interfaces and service methods beside that implementation, then +connect them to widget request handlers. Declare real credential alternatives, categories, Docker discovery aliases, +ports, and onboarding metadata in the local module instead of editing the central definitions and creator registries. + +The integration synchronizer validates kinds, credential kinds, categories, ports, URLs, documentation, creator +exports, and collisions. It generates the definitions projection, a typed lazy creator registry with literal imports, +and Docusaurus copies whose edit links point back to the canonical module docs. + +Beszel is migrated to this path. Other integrations still use the compatibility registry and can move independently +without changing persisted `IntegrationKind` values or widget capability checks. Keep integrations server-only, +implement only their capability interfaces, and use narrow schema/interface exports instead of the broad runtime +barrel while migrating them. diff --git a/apps/docs/docusaurus.config.ts b/apps/docs/docusaurus.config.ts index c20857c32..85a3e550f 100644 --- a/apps/docs/docusaurus.config.ts +++ b/apps/docs/docusaurus.config.ts @@ -2,12 +2,27 @@ import type * as Preset from "@docusaurus/preset-classic"; import type { Config } from "@docusaurus/types"; import { themes as prismThemes } from "prism-react-renderer"; import { resolveHomarrUrlConfig } from "@homarr/workshop/schema"; -import { generatedWidgetDocumentationSources } from "@homarr/definitions"; +import { generatedIntegrationDocumentationSources, generatedWidgetDocumentationSources } from "@homarr/definitions"; const a11yEmoji = require("@fec/remark-a11y-emoji"); const githubDocsEditRoot = "https://github.com/homarr-labs/homarr/edit/dev"; const getDocumentationEditUrl = (docPath: string) => { + const integrationPath = /^integrations\/([^/]+)\/(.+)$/u.exec(docPath); + const integrationSlug = integrationPath?.[1]; + const integrationRelativePath = integrationPath?.[2]; + if ( + integrationSlug && + integrationRelativePath && + Object.hasOwn(generatedIntegrationDocumentationSources, integrationSlug) + ) { + const source = + generatedIntegrationDocumentationSources[ + integrationSlug as keyof typeof generatedIntegrationDocumentationSources + ]; + return `${githubDocsEditRoot}/${source}/${integrationRelativePath}`; + } + const widgetPath = /^widgets\/([^/]+)\/(.+)$/u.exec(docPath); const slug = widgetPath?.[1]; const relativePath = widgetPath?.[2]; diff --git a/package.json b/package.json index ead235a09..0e18e96d2 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,9 @@ "docker:dev:up": "docker compose -f ./development/development.docker-compose.yml up -d redis", "format": "turbo format --continue -- --check", "format:fix": "turbo format --continue", + "features:sync": "pnpm widgets:sync && pnpm integrations:sync", + "features:sync:check": "pnpm widgets:sync:check && pnpm integrations:sync:check", + "integrations:new": "tsx ./scripts/integrations/create-integration-module.mts", "integrations:sync": "tsx ./scripts/integrations/sync-integration-modules.mts", "integrations:sync:check": "tsx ./scripts/integrations/sync-integration-modules.mts --check", "lint": "turbo lint --continue", diff --git a/packages/definitions/src/generated/integration-modules.ts b/packages/definitions/src/generated/integration-modules.ts index 19cd65e78..ad43793f9 100644 --- a/packages/definitions/src/generated/integration-modules.ts +++ b/packages/definitions/src/generated/integration-modules.ts @@ -10,3 +10,7 @@ export const generatedIntegrationDefinitions = { defaultPort: 8090, }, } as const; + +export const generatedIntegrationDocumentationSources = { + beszel: "packages/integrations/src/beszel/docs", +} as const; diff --git a/packages/redis/src/lib/channel.ts b/packages/redis/src/lib/channel.ts index f46d24c8c..ec49e82f4 100644 --- a/packages/redis/src/lib/channel.ts +++ b/packages/redis/src/lib/channel.ts @@ -170,28 +170,120 @@ export const createLockChannel = (name: string) => { const integrationCacheGenerationKey = (integrationId: string) => `integration-cache:generation:${integrationId}`; const pendingIntegrationCacheInvalidations = new Map(); const MAX_PENDING_INTEGRATION_CACHE_INVALIDATIONS = 1000; -const INTEGRATION_CACHE_REDIS_TIMEOUT_MS = 500; +const CACHE_GENERATION_REDIS_TIMEOUT_MS = 500; +const CACHE_GENERATION_SUCCESS_TTL_MS = 1000; +const CACHE_GENERATION_UNAVAILABLE_TTL_MS = 5000; +const MAX_CACHE_GENERATION_READS = 1000; + +export interface IntegrationCacheGeneration { + value: string; + isShared: boolean; +} + +interface CacheGenerationReadEntry { + generation: IntegrationCacheGeneration; + expiresAt: number; +} + +interface CacheGenerationReadInFlight { + promise: Promise; + token: object; +} + +const cacheGenerationReads = new Map(); +const cacheGenerationReadsInFlight = new Map(); const logCacheGenerationFailure = (message: string, metadata: Record, error: unknown) => { logger.warn(new ErrorWithMetadata(message, metadata, { cause: error })); }; -const withIntegrationCacheRedisTimeoutAsync = async (promise: Promise): Promise => { +const withCacheGenerationRedisTimeoutAsync = async (promise: Promise): Promise => { return await new Promise((resolve, reject) => { const timer = setTimeout( - () => reject(new Error("Redis integration cache operation timed out")), - INTEGRATION_CACHE_REDIS_TIMEOUT_MS, + () => reject(new Error("Redis cache generation operation timed out")), + CACHE_GENERATION_REDIS_TIMEOUT_MS, ); timer.unref?.(); void promise.then(resolve, reject).finally(() => clearTimeout(timer)); }); }; +const setCacheGenerationRead = ( + cacheKey: string, + generation: IntegrationCacheGeneration, + ttlMs = generation.isShared ? CACHE_GENERATION_SUCCESS_TTL_MS : CACHE_GENERATION_UNAVAILABLE_TTL_MS, +) => { + if (cacheGenerationReads.has(cacheKey)) cacheGenerationReads.delete(cacheKey); + while (cacheGenerationReads.size >= MAX_CACHE_GENERATION_READS) { + const oldestCacheKey = cacheGenerationReads.keys().next().value; + if (!oldestCacheKey) break; + cacheGenerationReads.delete(oldestCacheKey); + } + cacheGenerationReads.set(cacheKey, { + generation, + expiresAt: Date.now() + ttlMs, + }); +}; + +const beginCacheGenerationInvalidation = (cacheKey: string, pendingGeneration: string) => { + cacheGenerationReadsInFlight.delete(cacheKey); + setCacheGenerationRead(cacheKey, { value: pendingGeneration, isShared: false }); +}; + +interface GetCachedGenerationOptions { + cacheKey: string; + readAsync: () => Promise; + fallback: IntegrationCacheGeneration; + errorMessage: string; + errorMetadata: Record; + force?: boolean; +} + +const getCachedGenerationAsync = async ({ + cacheKey, + readAsync, + fallback, + errorMessage, + errorMetadata, + force = false, +}: GetCachedGenerationOptions): Promise => { + if (!force) { + const cached = cacheGenerationReads.get(cacheKey); + if (cached && Date.now() < cached.expiresAt) return cached.generation; + if (cached) cacheGenerationReads.delete(cacheKey); + } + + const existing = cacheGenerationReadsInFlight.get(cacheKey); + if (existing) return await existing.promise; + + const token = {}; + const promise = (async () => { + let generation = fallback; + try { + const value = await readAsync(); + if (value !== null) generation = { value, isShared: true }; + } catch (error) { + logCacheGenerationFailure(errorMessage, errorMetadata, error); + } + + if (cacheGenerationReadsInFlight.get(cacheKey)?.token === token) { + setCacheGenerationRead(cacheKey, generation); + } + return generation; + })().finally(() => { + if (cacheGenerationReadsInFlight.get(cacheKey)?.token === token) { + cacheGenerationReadsInFlight.delete(cacheKey); + } + }); + cacheGenerationReadsInFlight.set(cacheKey, { promise, token }); + return await promise; +}; + const advanceIntegrationCacheGenerationAsync = async (integrationId: string) => { const client = getSetClient as typeof getSetClient | null; if (!client) return null; - const results = await withIntegrationCacheRedisTimeoutAsync( + const results = await withCacheGenerationRedisTimeoutAsync( client.multi().incr(integrationCacheGenerationKey(integrationId)).del(`session-store:${integrationId}`).exec(), ); const generationResult = results?.[0]; @@ -201,54 +293,39 @@ const advanceIntegrationCacheGenerationAsync = async (integrationId: string) => return String(generationResult[1]); }; -export interface IntegrationCacheGeneration { - value: string; - isShared: boolean; -} - export const getIntegrationCacheGenerationAsync = async ( integrationId: string, ): Promise => { + const cacheKey = integrationCacheGenerationKey(integrationId); const pendingGeneration = pendingIntegrationCacheInvalidations.get(integrationId); if (pendingGeneration) { - try { - const generation = await advanceIntegrationCacheGenerationAsync(integrationId); - if (generation) { - if (pendingIntegrationCacheInvalidations.get(integrationId) === pendingGeneration) { - pendingIntegrationCacheInvalidations.delete(integrationId); - return { value: generation, isShared: true }; - } - return { - value: pendingIntegrationCacheInvalidations.get(integrationId) ?? generation, - isShared: false, - }; - } - } catch (error) { - logCacheGenerationFailure( - "Failed to retry integration cache invalidation", - { integrationId, operation: "generation-advance" }, - error, - ); + const generation = await getCachedGenerationAsync({ + cacheKey, + readAsync: async () => await advanceIntegrationCacheGenerationAsync(integrationId), + fallback: { value: pendingGeneration, isShared: false }, + errorMessage: "Failed to retry integration cache invalidation", + errorMetadata: { integrationId, operation: "generation-advance" }, + }); + if (generation.isShared && pendingIntegrationCacheInvalidations.get(integrationId) === pendingGeneration) { + pendingIntegrationCacheInvalidations.delete(integrationId); + return generation; } - return { value: pendingGeneration, isShared: false }; + const latestPendingGeneration = pendingIntegrationCacheInvalidations.get(integrationId); + if (latestPendingGeneration) return { value: latestPendingGeneration, isShared: false }; + return generation; } const client = getSetClient as typeof getSetClient | null; - if (!client) return { value: "redis-unavailable", isShared: false }; - - try { - const generation = await withIntegrationCacheRedisTimeoutAsync( - client.get(integrationCacheGenerationKey(integrationId)), - ); - return { value: generation ?? "0", isShared: true }; - } catch (error) { - logCacheGenerationFailure( - "Failed to read integration cache generation", - { integrationId, operation: "generation-read" }, - error, - ); - return { value: "redis-unavailable", isShared: false }; - } + return await getCachedGenerationAsync({ + cacheKey, + readAsync: async () => { + if (!client) return null; + return (await withCacheGenerationRedisTimeoutAsync(client.get(cacheKey))) ?? "0"; + }, + fallback: { value: "redis-unavailable", isShared: false }, + errorMessage: "Failed to read integration cache generation", + errorMetadata: { integrationId, operation: "generation-read" }, + }); }; export const invalidateIntegrationCacheAsync = async (integrationId: string): Promise => { @@ -258,18 +335,19 @@ export const invalidateIntegrationCacheAsync = async (integrationId: string): Pr if (oldestIntegrationId) pendingIntegrationCacheInvalidations.delete(oldestIntegrationId); } pendingIntegrationCacheInvalidations.set(integrationId, pendingGeneration); + const cacheKey = integrationCacheGenerationKey(integrationId); + beginCacheGenerationInvalidation(cacheKey, pendingGeneration); - try { - const generation = await advanceIntegrationCacheGenerationAsync(integrationId); - if (generation && pendingIntegrationCacheInvalidations.get(integrationId) === pendingGeneration) { - pendingIntegrationCacheInvalidations.delete(integrationId); - } - } catch (error) { - logCacheGenerationFailure( - "Failed to invalidate integration caches in Redis", - { integrationId, operation: "generation-advance-and-session-delete" }, - error, - ); + const generation = await getCachedGenerationAsync({ + cacheKey, + readAsync: async () => await advanceIntegrationCacheGenerationAsync(integrationId), + fallback: { value: pendingGeneration, isShared: false }, + errorMessage: "Failed to invalidate integration caches in Redis", + errorMetadata: { integrationId, operation: "generation-advance-and-session-delete" }, + force: true, + }); + if (generation.isShared && pendingIntegrationCacheInvalidations.get(integrationId) === pendingGeneration) { + pendingIntegrationCacheInvalidations.delete(integrationId); } }; @@ -281,50 +359,42 @@ const advanceWidgetCacheGenerationAsync = async (namespace: string) => { const client = getSetClient as typeof getSetClient | null; if (!client) return null; - return String(await withIntegrationCacheRedisTimeoutAsync(client.incr(widgetCacheGenerationKey(namespace)))); + return String(await withCacheGenerationRedisTimeoutAsync(client.incr(widgetCacheGenerationKey(namespace)))); }; export type WidgetCacheGeneration = IntegrationCacheGeneration; export const getWidgetCacheGenerationAsync = async (namespace: string): Promise => { + const cacheKey = widgetCacheGenerationKey(namespace); const pendingGeneration = pendingWidgetCacheInvalidations.get(namespace); if (pendingGeneration) { - try { - const generation = await advanceWidgetCacheGenerationAsync(namespace); - if (generation) { - if (pendingWidgetCacheInvalidations.get(namespace) === pendingGeneration) { - pendingWidgetCacheInvalidations.delete(namespace); - return { value: generation, isShared: true }; - } - return { - value: pendingWidgetCacheInvalidations.get(namespace) ?? generation, - isShared: false, - }; - } - } catch (error) { - logCacheGenerationFailure( - "Failed to retry widget cache invalidation", - { cacheNamespace: namespace, operation: "generation-advance" }, - error, - ); + const generation = await getCachedGenerationAsync({ + cacheKey, + readAsync: async () => await advanceWidgetCacheGenerationAsync(namespace), + fallback: { value: pendingGeneration, isShared: false }, + errorMessage: "Failed to retry widget cache invalidation", + errorMetadata: { cacheNamespace: namespace, operation: "generation-advance" }, + }); + if (generation.isShared && pendingWidgetCacheInvalidations.get(namespace) === pendingGeneration) { + pendingWidgetCacheInvalidations.delete(namespace); + return generation; } - return { value: pendingGeneration, isShared: false }; + const latestPendingGeneration = pendingWidgetCacheInvalidations.get(namespace); + if (latestPendingGeneration) return { value: latestPendingGeneration, isShared: false }; + return generation; } const client = getSetClient as typeof getSetClient | null; - if (!client) return { value: "redis-unavailable", isShared: false }; - - try { - const generation = await withIntegrationCacheRedisTimeoutAsync(client.get(widgetCacheGenerationKey(namespace))); - return { value: generation ?? "0", isShared: true }; - } catch (error) { - logCacheGenerationFailure( - "Failed to read widget cache generation", - { cacheNamespace: namespace, operation: "generation-read" }, - error, - ); - return { value: "redis-unavailable", isShared: false }; - } + return await getCachedGenerationAsync({ + cacheKey, + readAsync: async () => { + if (!client) return null; + return (await withCacheGenerationRedisTimeoutAsync(client.get(cacheKey))) ?? "0"; + }, + fallback: { value: "redis-unavailable", isShared: false }, + errorMessage: "Failed to read widget cache generation", + errorMetadata: { cacheNamespace: namespace, operation: "generation-read" }, + }); }; export const invalidateWidgetCache = (namespace: string): void => { @@ -334,20 +404,21 @@ export const invalidateWidgetCache = (namespace: string): void => { if (oldestNamespace) pendingWidgetCacheInvalidations.delete(oldestNamespace); } pendingWidgetCacheInvalidations.set(namespace, pendingGeneration); + const cacheKey = widgetCacheGenerationKey(namespace); + beginCacheGenerationInvalidation(cacheKey, pendingGeneration); - void advanceWidgetCacheGenerationAsync(namespace) - .then((generation) => { - if (generation && pendingWidgetCacheInvalidations.get(namespace) === pendingGeneration) { - pendingWidgetCacheInvalidations.delete(namespace); - } - }) - .catch((error: unknown) => { - logCacheGenerationFailure( - "Failed to invalidate widget caches in Redis", - { cacheNamespace: namespace, operation: "generation-advance" }, - error, - ); - }); + void getCachedGenerationAsync({ + cacheKey, + readAsync: async () => await advanceWidgetCacheGenerationAsync(namespace), + fallback: { value: pendingGeneration, isShared: false }, + errorMessage: "Failed to invalidate widget caches in Redis", + errorMetadata: { cacheNamespace: namespace, operation: "generation-advance" }, + force: true, + }).then((generation) => { + if (generation.isShared && pendingWidgetCacheInvalidations.get(namespace) === pendingGeneration) { + pendingWidgetCacheInvalidations.delete(namespace); + } + }); }; /** diff --git a/scripts/integrations/create-integration-module.mts b/scripts/integrations/create-integration-module.mts new file mode 100644 index 000000000..5f725ac92 --- /dev/null +++ b/scripts/integrations/create-integration-module.mts @@ -0,0 +1,120 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const integrationsRoot = path.join(repositoryRoot, "packages/integrations/src"); + +const readOption = (name: string) => { + const optionIndex = process.argv.indexOf(name); + if (optionIndex < 0) return undefined; + return process.argv[optionIndex + 1]; +}; + +const kind = process.argv[2]; +if (!kind || kind.startsWith("--")) { + throw new Error( + "Usage: pnpm integrations:new --icon-url [--slug ] [--name ] [--category ] [--default-port ]", + ); +} +if (!/^[a-z][A-Za-z0-9]*$/.test(kind)) { + throw new Error("Integration kind must be a stable lower-camel-case identifier"); +} + +const defaultSlug = kind.replace(/([a-z0-9])([A-Z])/gu, "$1-$2").toLowerCase(); +const slug = readOption("--slug") ?? defaultSlug; +const name = + readOption("--name") ?? + slug + .split("-") + .map((part) => `${part.charAt(0).toUpperCase()}${part.slice(1)}`) + .join(" "); +const category = readOption("--category") ?? "miscellaneous"; +const iconUrl = readOption("--icon-url"); +const defaultPortInput = readOption("--default-port"); + +if (!/^[a-z0-9][a-z0-9-]*$/.test(slug)) throw new Error("Integration slug must use lower-case kebab-case"); +if (!iconUrl || !URL.canParse(iconUrl)) throw new Error("--icon-url must be a valid absolute URL"); +if (!/^[a-z][A-Za-z0-9]*$/.test(category)) throw new Error("Integration category must be a stable identifier"); + +let defaultPort: number | undefined; +if (defaultPortInput !== undefined) { + defaultPort = Number(defaultPortInput); + if (!Number.isSafeInteger(defaultPort) || defaultPort < 1 || defaultPort > 65_535) { + throw new Error("--default-port must be an integer between 1 and 65535"); + } +} + +const className = `${kind.charAt(0).toUpperCase()}${kind.slice(1)}Integration`; +const metadataExportName = `${kind}Integration`; +const directory = path.join(integrationsRoot, slug); +await mkdir(directory); +await mkdir(path.join(directory, "docs")); + +const defaultPortLine = defaultPort === undefined ? "" : ` defaultPort: ${defaultPort},\n`; +const files = { + "module.ts": `import { defineIntegrationModule } from "@homarr/definitions"; + +export default defineIntegrationModule({ + kind: ${JSON.stringify(kind)}, + name: ${JSON.stringify(name)}, + iconUrl: ${JSON.stringify(iconUrl)}, + secretKinds: [[]], + categories: [${JSON.stringify(category)}], +${defaultPortLine} documentation: { + slug: ${JSON.stringify(slug)}, + sourceDirectory: "docs", + }, + creator: { + type: "constructor", + module: "./${slug}-integration", + exportName: ${JSON.stringify(className)}, + }, +}); +`, + [`${slug}-integration.ts`]: `import type { IntegrationTestingInput } from "../base/integration"; +import { Integration } from "../base/integration"; +import { TestConnectionError } from "../base/test-connection/test-connection-error"; +import type { TestingResult } from "../base/test-connection/test-connection-service"; + +export class ${className} extends Integration { + protected async testingAsync(input: IntegrationTestingInput): Promise { + const response = await input.fetchAsync(this.url("/")); + if (!response.ok) return TestConnectionError.StatusResult(response); + return { success: true }; + } +} +`, + "docs/index.ts": `export const ${metadataExportName} = { + name: ${JSON.stringify(name)}, + description: ${JSON.stringify(`Connect Homarr to ${name}.`)}, + iconUrl: ${JSON.stringify(iconUrl)}, + path: "../../integrations/${slug}", +} as const; +`, + "docs/index.mdx": `--- +title: ${JSON.stringify(name)} +description: ${JSON.stringify(`Connect Homarr to ${name}.`)} +hide_title: true +--- + +import { IntegrationHeader } from "@site/src/components/integrations/header"; +import { AddingIntegration } from "@site/src/components/integrations/adding"; +import { ${metadataExportName} } from "."; + + + +Describe the service, supported Homarr widgets, required permissions, and setup prerequisites here. + +### Adding the integration + + +`, +} as const; + +for (const [relativePath, content] of Object.entries(files)) { + await writeFile(path.join(directory, relativePath), content, { flag: "wx" }); +} + +await import("./sync-integration-modules.mts"); +console.log(`Created integration module ${kind} in ${path.relative(repositoryRoot, directory)}`); diff --git a/scripts/integrations/sync-integration-modules.mts b/scripts/integrations/sync-integration-modules.mts index 959798356..4be477ea0 100644 --- a/scripts/integrations/sync-integration-modules.mts +++ b/scripts/integrations/sync-integration-modules.mts @@ -5,6 +5,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { format } from "oxfmt"; import type { IntegrationModuleDefinition } from "../../packages/definitions/src/integration-module"; +import { integrationCategories, integrationSecretKinds } from "../../packages/definitions/src/integration"; const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const integrationsRoot = path.join(repositoryRoot, "packages/integrations/src"); @@ -15,6 +16,8 @@ const integrationCreatorsSource = path.join(integrationsRoot, "base/creator.ts") const checkOnly = process.argv.includes("--check"); const generatedHeader = "// This file is generated by pnpm integrations:sync. Do not edit it directly.\n\n"; const generatedDocsMarker = ".integration-module-generated"; +const integrationCategorySet = new Set(integrationCategories); +const integrationSecretKindSet = new Set(integrationSecretKinds); interface LoadedIntegrationModule { definition: IntegrationModuleDefinition; @@ -106,17 +109,35 @@ const validateIntegrationModuleAsync = async (module: LoadedIntegrationModule) = if (!Array.isArray(definition.secretKinds) || definition.secretKinds.length === 0) { throw new Error(`Integration ${definition.kind} must declare at least one secret alternative`); } + for (const secretAlternative of definition.secretKinds) { + if (!Array.isArray(secretAlternative)) { + throw new Error(`Integration ${definition.kind} secret alternatives must be arrays`); + } + for (const secretKind of secretAlternative) { + if (!integrationSecretKindSet.has(secretKind)) { + throw new Error(`Integration ${definition.kind} has unknown secret kind ${secretKind}`); + } + } + } if (!Array.isArray(definition.categories) || definition.categories.length === 0) { throw new Error(`Integration ${definition.kind} must declare at least one category`); } + for (const category of definition.categories) { + if (!integrationCategorySet.has(category)) { + throw new Error(`Integration ${definition.kind} has unknown category ${category}`); + } + } if (!/^[a-z0-9][a-z0-9-]*$/.test(definition.documentation.slug)) { throw new Error(`Integration ${definition.kind} has invalid documentation slug ${definition.documentation.slug}`); } if ( definition.defaultPort !== undefined && - (!Number.isSafeInteger(definition.defaultPort) || definition.defaultPort < 1) + (!Number.isSafeInteger(definition.defaultPort) || definition.defaultPort < 1 || definition.defaultPort > 65_535) ) { - throw new Error(`Integration ${definition.kind} default port must be a positive integer`); + throw new Error(`Integration ${definition.kind} default port must be an integer between 1 and 65535`); + } + if (definition.defaultUrl !== undefined && !URL.canParse(definition.defaultUrl)) { + throw new Error(`Integration ${definition.kind} default URL is invalid`); } const docsDirectory = path.resolve(module.directory, definition.documentation.sourceDirectory); @@ -231,7 +252,14 @@ const renderDefinitionsRegistry = (modules: LoadedIntegrationModule[]) => { const entries = modules .map((module) => ` ${JSON.stringify(module.definition.kind)}: ${renderDefinition(module.definition)},`) .join("\n"); - return `${generatedHeader}export const generatedIntegrationDefinitions = {\n${entries}\n} as const;\n`; + const documentationSources = modules + .map((module) => { + const source = path.resolve(module.directory, module.definition.documentation.sourceDirectory); + const relativeSource = toPosixPath(path.relative(repositoryRoot, source)); + return ` ${JSON.stringify(module.definition.documentation.slug)}: ${JSON.stringify(relativeSource)},`; + }) + .join("\n"); + return `${generatedHeader}export const generatedIntegrationDefinitions = {\n${entries}\n} as const;\n\nexport const generatedIntegrationDocumentationSources = {\n${documentationSources}\n} as const;\n`; }; const getCreatorImportPath = (module: LoadedIntegrationModule) => {