import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { format } from "oxfmt"; import type { IntegrationModuleDefinition } from "../../packages/definitions/src/integration-module"; const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const integrationsRoot = path.join(repositoryRoot, "packages/integrations/src"); const generatedRoot = path.join(integrationsRoot, "generated"); const docsRoot = path.join(repositoryRoot, "apps/docs/docs/integrations"); const integrationDefinitionsSource = path.join(repositoryRoot, "packages/definitions/src/integration.ts"); const integrationCreatorsSource = path.join(integrationsRoot, "base/creator.ts"); const checkOnly = process.argv.includes("--check"); const generatedHeader = "// This file is generated by pnpm integrations:sync. Do not edit it directly.\n\n"; const generatedDocsMarker = ".integration-module-generated"; interface LoadedIntegrationModule { definition: IntegrationModuleDefinition; directory: string; } interface GeneratedFile { path: string; content: Buffer; } const toPosixPath = (value: string) => value.split(path.sep).join("/"); const pathExistsAsync = async (target: string) => { try { await stat(target); return true; } catch { return false; } }; const assertPathInside = (parent: string, target: string, label: string) => { const relative = path.relative(parent, target); if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) return; throw new Error(`${label} must stay inside ${parent}`); }; const findSourceEntryAsync = async (target: string) => { const candidates = [ target, `${target}.ts`, `${target}.tsx`, path.join(target, "index.ts"), path.join(target, "index.tsx"), ]; for (const candidate of candidates) { if (await pathExistsAsync(candidate)) return candidate; } return null; }; const findIntegrationModuleFilesAsync = async (directory: string): Promise => { const entries = await readdir(directory, { withFileTypes: true }); const files: string[] = []; for (const entry of entries) { if (entry.name === "generated" || entry.name === "docs" || entry.name.startsWith(".")) continue; const target = path.join(directory, entry.name); if (entry.isDirectory()) { files.push(...(await findIntegrationModuleFilesAsync(target))); continue; } if (entry.isFile() && entry.name === "module.ts") files.push(target); } return files; }; const isRecord = (value: unknown): value is Record => value !== null && typeof value === "object" && !Array.isArray(value); const assertNonEmptyString = (value: unknown, label: string): asserts value is string => { if (typeof value !== "string" || value.trim() === "") throw new Error(`${label} must be a non-empty string`); }; const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const validateExportAsync = async (sourcePath: string, exportName: string, kind: string) => { const source = await readFile(sourcePath, "utf8"); const escapedName = escapeRegExp(exportName); const declaration = new RegExp( `export\\s+(?:declare\\s+)?(?:abstract\\s+)?(?:class|const|function)\\s+${escapedName}\\b`, ); const exportList = new RegExp(`export\\s*\\{[^}]*\\b${escapedName}\\b[^}]*\\}`, "s"); if (declaration.test(source) || exportList.test(source)) return; throw new Error(`Integration ${kind} creator export ${exportName} was not found in ${sourcePath}`); }; const validateIntegrationModuleAsync = async (module: LoadedIntegrationModule) => { const { definition } = module; assertNonEmptyString(definition.kind, "Integration kind"); if (!/^[A-Za-z][A-Za-z0-9]*$/.test(definition.kind)) { throw new Error(`Integration kind ${definition.kind} contains unsupported characters`); } assertNonEmptyString(definition.name, `Integration ${definition.kind} name`); assertNonEmptyString(definition.iconUrl, `Integration ${definition.kind} icon URL`); if (!URL.canParse(definition.iconUrl)) throw new Error(`Integration ${definition.kind} icon URL is invalid`); if (!Array.isArray(definition.secretKinds) || definition.secretKinds.length === 0) { throw new Error(`Integration ${definition.kind} must declare at least one secret alternative`); } if (!Array.isArray(definition.categories) || definition.categories.length === 0) { throw new Error(`Integration ${definition.kind} must declare at least one category`); } if (!/^[a-z0-9][a-z0-9-]*$/.test(definition.documentation.slug)) { throw new Error(`Integration ${definition.kind} has invalid documentation slug ${definition.documentation.slug}`); } if ( definition.defaultPort !== undefined && (!Number.isSafeInteger(definition.defaultPort) || definition.defaultPort < 1) ) { throw new Error(`Integration ${definition.kind} default port must be a positive integer`); } const docsDirectory = path.resolve(module.directory, definition.documentation.sourceDirectory); assertPathInside(module.directory, docsDirectory, `Integration ${definition.kind} documentation`); if (!(await pathExistsAsync(path.join(docsDirectory, "index.mdx")))) { throw new Error(`Integration ${definition.kind} documentation must contain index.mdx: ${docsDirectory}`); } if (!(await pathExistsAsync(path.join(docsDirectory, "index.ts")))) { throw new Error(`Integration ${definition.kind} documentation must contain index.ts: ${docsDirectory}`); } const creatorTarget = path.resolve(module.directory, definition.creator.module); assertPathInside(module.directory, creatorTarget, `Integration ${definition.kind} creator`); const creatorSource = await findSourceEntryAsync(creatorTarget); if (!creatorSource) { throw new Error(`Integration ${definition.kind} creator module does not exist: ${definition.creator.module}`); } assertNonEmptyString(definition.creator.exportName, `Integration ${definition.kind} creator export`); await validateExportAsync(creatorSource, definition.creator.exportName, definition.kind); }; const loadHandAuthoredRegistryEntriesAsync = async () => { const definitionsSource = await readFile(integrationDefinitionsSource, "utf8"); const startMarker = "createIntegrationDefinitions({"; const endMarker = "} as const satisfies Record);"; const start = definitionsSource.indexOf(startMarker); const end = definitionsSource.indexOf(endMarker, start); if (start < 0 || end < 0) throw new Error("Unable to locate the hand-authored integration registry"); const registry = definitionsSource.slice(start + startMarker.length, end); const kinds = new Set([...registry.matchAll(/^ ([A-Za-z][A-Za-z0-9]*): \{/gm)].map((match) => match[1])); const docsSlugs = new Set( [...registry.matchAll(/^ documentationSlug: "([a-z0-9][a-z0-9-]*)",/gm)].map((match) => match[1]), ); const creatorsSource = await readFile(integrationCreatorsSource, "utf8"); const creatorStartMarker = "const integrationCreatorLoaders = {"; const creatorEndMarker = "\n};\n\ntype AssertIntegrationCreatorLoaders"; const creatorStart = creatorsSource.indexOf(creatorStartMarker); const creatorEnd = creatorsSource.indexOf(creatorEndMarker, creatorStart); if (creatorStart < 0 || creatorEnd < 0) throw new Error("Unable to locate the hand-authored creator registry"); const creatorRegistry = creatorsSource.slice(creatorStart + creatorStartMarker.length, creatorEnd); const creatorKinds = new Set( [...creatorRegistry.matchAll(/^ ([A-Za-z][A-Za-z0-9]*): async /gm)].map((match) => match[1]), ); return { kinds, docsSlugs, creatorKinds }; }; const loadIntegrationModulesAsync = async () => { const moduleFiles = (await findIntegrationModuleFilesAsync(integrationsRoot)).toSorted(); const handAuthored = await loadHandAuthoredRegistryEntriesAsync(); const modules: LoadedIntegrationModule[] = []; const kinds = new Set(); const docsSlugs = new Set(); for (const moduleFile of moduleFiles) { const loaded: unknown = await import(`${pathToFileURL(moduleFile).href}?sync=${Date.now()}`); if (!isRecord(loaded) || !("default" in loaded) || !isRecord(loaded.default)) { throw new Error(`${moduleFile} must default-export defineIntegrationModule(...)`); } const directory = path.dirname(moduleFile); const module: LoadedIntegrationModule = { definition: loaded.default as unknown as IntegrationModuleDefinition, directory, }; await validateIntegrationModuleAsync(module); if (handAuthored.kinds.has(module.definition.kind)) { throw new Error(`Integration kind ${module.definition.kind} exists in both a module and the central registry`); } if (handAuthored.creatorKinds.has(module.definition.kind)) { throw new Error(`Integration kind ${module.definition.kind} exists in both a module and the central creator`); } if (handAuthored.docsSlugs.has(module.definition.documentation.slug)) { throw new Error( `Integration documentation slug ${module.definition.documentation.slug} exists in both a module and the central registry`, ); } if (kinds.has(module.definition.kind)) throw new Error(`Duplicate integration kind ${module.definition.kind}`); kinds.add(module.definition.kind); if (docsSlugs.has(module.definition.documentation.slug)) { throw new Error(`Duplicate integration documentation slug ${module.definition.documentation.slug}`); } docsSlugs.add(module.definition.documentation.slug); modules.push(module); } return modules.toSorted((left, right) => left.definition.kind.localeCompare(right.definition.kind)); }; const renderDefinition = (definition: IntegrationModuleDefinition) => { const projected: Record = { name: definition.name, secretKinds: definition.secretKinds, iconUrl: definition.iconUrl, category: definition.categories, documentationSlug: definition.documentation.slug, }; if (definition.defaultUrl !== undefined) projected.defaultUrl = definition.defaultUrl; if (definition.defaultPort !== undefined) projected.defaultPort = definition.defaultPort; if (definition.apiKeySettingsPath !== undefined) projected.apiKeySettingsPath = definition.apiKeySettingsPath; if (definition.docker || definition.onboarding) { const features: Record = {}; if (definition.docker) features.docker = definition.docker; if (definition.onboarding) features.onboarding = definition.onboarding; projected.features = features; } return JSON.stringify(projected, null, 2); }; const renderDefinitionsRegistry = (modules: LoadedIntegrationModule[]) => { const entries = modules .map((module) => ` ${JSON.stringify(module.definition.kind)}: ${renderDefinition(module.definition)},`) .join("\n"); return `${generatedHeader}export const generatedIntegrationDefinitions = {\n${entries}\n} as const;\n`; }; const getCreatorImportPath = (module: LoadedIntegrationModule) => { const target = path.resolve(module.directory, module.definition.creator.module); const relative = toPosixPath(path.relative(generatedRoot, target)); if (relative.startsWith(".")) return relative; return `./${relative}`; }; const renderCreatorLoader = (module: LoadedIntegrationModule) => { const { creator, kind } = module.definition; const importPath = getCreatorImportPath(module); if (creator.type === "factory") { return ` ${JSON.stringify(kind)}: async () => (await import(${JSON.stringify(importPath)})).${creator.exportName},`; } return ` ${JSON.stringify(kind)}: async () => {\n const IntegrationClass = (await import(${JSON.stringify(importPath)})).${creator.exportName};\n return async (input: IntegrationInput) => new IntegrationClass(input);\n },`; }; const renderCreatorRegistry = (modules: LoadedIntegrationModule[]) => { const entries = modules.map(renderCreatorLoader).join("\n"); return `${generatedHeader}import type { Integration, IntegrationInput } from "../base/integration";\n\ntype GeneratedIntegrationCreator = (input: IntegrationInput) => Promise;\n\nexport const generatedIntegrationCreatorLoaders = {\n${entries}\n} satisfies Record Promise>;\n`; }; const collectFilesAsync = async (directory: string, relativeDirectory = ""): Promise => { const entries = await readdir(directory, { withFileTypes: true }); const files: GeneratedFile[] = []; for (const entry of entries) { const relativePath = path.join(relativeDirectory, entry.name); const sourcePath = path.join(directory, entry.name); if (entry.isDirectory()) { files.push(...(await collectFilesAsync(sourcePath, relativePath))); continue; } if (entry.isFile()) files.push({ path: relativePath, content: await readFile(sourcePath) }); } return files; }; const createDocsOutputsAsync = async (modules: LoadedIntegrationModule[]) => { const outputs: GeneratedFile[] = []; for (const module of modules) { const source = path.resolve(module.directory, module.definition.documentation.sourceDirectory); const target = path.join(docsRoot, module.definition.documentation.slug); for (const file of await collectFilesAsync(source)) { outputs.push({ path: path.join(target, file.path), content: file.content }); } outputs.push({ path: path.join(target, generatedDocsMarker), content: Buffer.from(`${toPosixPath(path.relative(repositoryRoot, source))}\n`), }); } return outputs; }; const readOrEmptyAsync = async (target: string) => { try { return await readFile(target); } catch { return null; } }; const createGeneratedTypeScriptAsync = async (target: string, source: string): Promise => { const result = await format(target, source, { printWidth: 120, endOfLine: "lf" }); const errors = result.errors.filter((error) => error.severity === "Error"); if (errors.length > 0) { throw new Error(`Failed to format ${target}: ${errors.map((error) => error.message).join(", ")}`); } return { path: target, content: Buffer.from(result.code) }; }; const synchronizeFileAsync = async (file: GeneratedFile) => { const current = await readOrEmptyAsync(file.path); if (current?.equals(file.content)) return false; if (checkOnly) return true; await mkdir(path.dirname(file.path), { recursive: true }); await writeFile(file.path, file.content); return true; }; const removeStaleGeneratedDocsAsync = async (outputs: GeneratedFile[]) => { const expected = new Set(outputs.map((output) => path.resolve(output.path))); const stale: string[] = []; const entries = await readdir(docsRoot, { withFileTypes: true }); for (const entry of entries) { if (!entry.isDirectory()) continue; const target = path.join(docsRoot, entry.name); if (!(await pathExistsAsync(path.join(target, generatedDocsMarker)))) continue; for (const file of await collectFilesAsync(target)) { const absolutePath = path.resolve(target, file.path); if (!expected.has(absolutePath)) stale.push(absolutePath); } } if (!checkOnly) { for (const file of stale) await rm(file); } return stale; }; const mainAsync = async () => { const modules = await loadIntegrationModulesAsync(); if (modules.length === 0) throw new Error("No vertical integration modules were found"); const definitionsTarget = path.join(repositoryRoot, "packages/definitions/src/generated/integration-modules.ts"); const creatorsTarget = path.join(generatedRoot, "integration-creators.ts"); const outputs: GeneratedFile[] = [ await createGeneratedTypeScriptAsync(definitionsTarget, renderDefinitionsRegistry(modules)), await createGeneratedTypeScriptAsync(creatorsTarget, renderCreatorRegistry(modules)), ...(await createDocsOutputsAsync(modules)), ]; const staleDocs = await removeStaleGeneratedDocsAsync(outputs); const drifted: string[] = []; for (const output of outputs) { if (await synchronizeFileAsync(output)) drifted.push(path.relative(repositoryRoot, output.path)); } drifted.push(...staleDocs.map((file) => path.relative(repositoryRoot, file))); if (checkOnly && drifted.length > 0) { throw new Error( `Integration module artifacts are out of date:\n${drifted.map((file) => ` - ${file}`).join("\n")}\nRun pnpm integrations:sync.`, ); } let action = "synchronized"; if (checkOnly) action = "verified"; console.log(`Integration modules ${action}: ${modules.map((module) => module.definition.kind).join(", ")}`); }; await mainAsync();