365 lines
16 KiB
TypeScript
365 lines
16 KiB
TypeScript
import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
|
|
import { format } from "oxfmt";
|
|
|
|
import type { IntegrationModuleDefinition } from "../../packages/definitions/src/integration-module";
|
|
|
|
const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const integrationsRoot = path.join(repositoryRoot, "packages/integrations/src");
|
|
const generatedRoot = path.join(integrationsRoot, "generated");
|
|
const docsRoot = path.join(repositoryRoot, "apps/docs/docs/integrations");
|
|
const integrationDefinitionsSource = path.join(repositoryRoot, "packages/definitions/src/integration.ts");
|
|
const integrationCreatorsSource = path.join(integrationsRoot, "base/creator.ts");
|
|
const checkOnly = process.argv.includes("--check");
|
|
const generatedHeader = "// This file is generated by pnpm integrations:sync. Do not edit it directly.\n\n";
|
|
const generatedDocsMarker = ".integration-module-generated";
|
|
|
|
interface LoadedIntegrationModule {
|
|
definition: IntegrationModuleDefinition;
|
|
directory: string;
|
|
}
|
|
|
|
interface GeneratedFile {
|
|
path: string;
|
|
content: Buffer;
|
|
}
|
|
|
|
const toPosixPath = (value: string) => value.split(path.sep).join("/");
|
|
|
|
const pathExistsAsync = async (target: string) => {
|
|
try {
|
|
await stat(target);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
};
|
|
|
|
const assertPathInside = (parent: string, target: string, label: string) => {
|
|
const relative = path.relative(parent, target);
|
|
if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) return;
|
|
throw new Error(`${label} must stay inside ${parent}`);
|
|
};
|
|
|
|
const findSourceEntryAsync = async (target: string) => {
|
|
const candidates = [
|
|
target,
|
|
`${target}.ts`,
|
|
`${target}.tsx`,
|
|
path.join(target, "index.ts"),
|
|
path.join(target, "index.tsx"),
|
|
];
|
|
for (const candidate of candidates) {
|
|
if (await pathExistsAsync(candidate)) return candidate;
|
|
}
|
|
return null;
|
|
};
|
|
|
|
const findIntegrationModuleFilesAsync = async (directory: string): Promise<string[]> => {
|
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
const files: string[] = [];
|
|
|
|
for (const entry of entries) {
|
|
if (entry.name === "generated" || entry.name === "docs" || entry.name.startsWith(".")) continue;
|
|
const target = path.join(directory, entry.name);
|
|
if (entry.isDirectory()) {
|
|
files.push(...(await findIntegrationModuleFilesAsync(target)));
|
|
continue;
|
|
}
|
|
if (entry.isFile() && entry.name === "module.ts") files.push(target);
|
|
}
|
|
|
|
return files;
|
|
};
|
|
|
|
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
value !== null && typeof value === "object" && !Array.isArray(value);
|
|
|
|
const assertNonEmptyString = (value: unknown, label: string): asserts value is string => {
|
|
if (typeof value !== "string" || value.trim() === "") throw new Error(`${label} must be a non-empty string`);
|
|
};
|
|
|
|
const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
|
|
const validateExportAsync = async (sourcePath: string, exportName: string, kind: string) => {
|
|
const source = await readFile(sourcePath, "utf8");
|
|
const escapedName = escapeRegExp(exportName);
|
|
const declaration = new RegExp(
|
|
`export\\s+(?:declare\\s+)?(?:abstract\\s+)?(?:class|const|function)\\s+${escapedName}\\b`,
|
|
);
|
|
const exportList = new RegExp(`export\\s*\\{[^}]*\\b${escapedName}\\b[^}]*\\}`, "s");
|
|
if (declaration.test(source) || exportList.test(source)) return;
|
|
throw new Error(`Integration ${kind} creator export ${exportName} was not found in ${sourcePath}`);
|
|
};
|
|
|
|
const validateIntegrationModuleAsync = async (module: LoadedIntegrationModule) => {
|
|
const { definition } = module;
|
|
assertNonEmptyString(definition.kind, "Integration kind");
|
|
if (!/^[A-Za-z][A-Za-z0-9]*$/.test(definition.kind)) {
|
|
throw new Error(`Integration kind ${definition.kind} contains unsupported characters`);
|
|
}
|
|
assertNonEmptyString(definition.name, `Integration ${definition.kind} name`);
|
|
assertNonEmptyString(definition.iconUrl, `Integration ${definition.kind} icon URL`);
|
|
if (!URL.canParse(definition.iconUrl)) throw new Error(`Integration ${definition.kind} icon URL is invalid`);
|
|
if (!Array.isArray(definition.secretKinds) || definition.secretKinds.length === 0) {
|
|
throw new Error(`Integration ${definition.kind} must declare at least one secret alternative`);
|
|
}
|
|
if (!Array.isArray(definition.categories) || definition.categories.length === 0) {
|
|
throw new Error(`Integration ${definition.kind} must declare at least one category`);
|
|
}
|
|
if (!/^[a-z0-9][a-z0-9-]*$/.test(definition.documentation.slug)) {
|
|
throw new Error(`Integration ${definition.kind} has invalid documentation slug ${definition.documentation.slug}`);
|
|
}
|
|
if (
|
|
definition.defaultPort !== undefined &&
|
|
(!Number.isSafeInteger(definition.defaultPort) || definition.defaultPort < 1)
|
|
) {
|
|
throw new Error(`Integration ${definition.kind} default port must be a positive integer`);
|
|
}
|
|
|
|
const docsDirectory = path.resolve(module.directory, definition.documentation.sourceDirectory);
|
|
assertPathInside(module.directory, docsDirectory, `Integration ${definition.kind} documentation`);
|
|
if (!(await pathExistsAsync(path.join(docsDirectory, "index.mdx")))) {
|
|
throw new Error(`Integration ${definition.kind} documentation must contain index.mdx: ${docsDirectory}`);
|
|
}
|
|
if (!(await pathExistsAsync(path.join(docsDirectory, "index.ts")))) {
|
|
throw new Error(`Integration ${definition.kind} documentation must contain index.ts: ${docsDirectory}`);
|
|
}
|
|
|
|
const creatorTarget = path.resolve(module.directory, definition.creator.module);
|
|
assertPathInside(module.directory, creatorTarget, `Integration ${definition.kind} creator`);
|
|
const creatorSource = await findSourceEntryAsync(creatorTarget);
|
|
if (!creatorSource) {
|
|
throw new Error(`Integration ${definition.kind} creator module does not exist: ${definition.creator.module}`);
|
|
}
|
|
assertNonEmptyString(definition.creator.exportName, `Integration ${definition.kind} creator export`);
|
|
await validateExportAsync(creatorSource, definition.creator.exportName, definition.kind);
|
|
};
|
|
|
|
const loadHandAuthoredRegistryEntriesAsync = async () => {
|
|
const definitionsSource = await readFile(integrationDefinitionsSource, "utf8");
|
|
const startMarker = "createIntegrationDefinitions({";
|
|
const endMarker = "} as const satisfies Record<string, integrationDefinition>);";
|
|
const start = definitionsSource.indexOf(startMarker);
|
|
const end = definitionsSource.indexOf(endMarker, start);
|
|
if (start < 0 || end < 0) throw new Error("Unable to locate the hand-authored integration registry");
|
|
|
|
const registry = definitionsSource.slice(start + startMarker.length, end);
|
|
const kinds = new Set([...registry.matchAll(/^ ([A-Za-z][A-Za-z0-9]*): \{/gm)].map((match) => match[1]));
|
|
const docsSlugs = new Set(
|
|
[...registry.matchAll(/^ documentationSlug: "([a-z0-9][a-z0-9-]*)",/gm)].map((match) => match[1]),
|
|
);
|
|
|
|
const creatorsSource = await readFile(integrationCreatorsSource, "utf8");
|
|
const creatorStartMarker = "const integrationCreatorLoaders = {";
|
|
const creatorEndMarker = "\n};\n\ntype AssertIntegrationCreatorLoaders";
|
|
const creatorStart = creatorsSource.indexOf(creatorStartMarker);
|
|
const creatorEnd = creatorsSource.indexOf(creatorEndMarker, creatorStart);
|
|
if (creatorStart < 0 || creatorEnd < 0) throw new Error("Unable to locate the hand-authored creator registry");
|
|
const creatorRegistry = creatorsSource.slice(creatorStart + creatorStartMarker.length, creatorEnd);
|
|
const creatorKinds = new Set(
|
|
[...creatorRegistry.matchAll(/^ ([A-Za-z][A-Za-z0-9]*): async /gm)].map((match) => match[1]),
|
|
);
|
|
return { kinds, docsSlugs, creatorKinds };
|
|
};
|
|
|
|
const loadIntegrationModulesAsync = async () => {
|
|
const moduleFiles = (await findIntegrationModuleFilesAsync(integrationsRoot)).toSorted();
|
|
const handAuthored = await loadHandAuthoredRegistryEntriesAsync();
|
|
const modules: LoadedIntegrationModule[] = [];
|
|
const kinds = new Set<string>();
|
|
const docsSlugs = new Set<string>();
|
|
|
|
for (const moduleFile of moduleFiles) {
|
|
const loaded: unknown = await import(`${pathToFileURL(moduleFile).href}?sync=${Date.now()}`);
|
|
if (!isRecord(loaded) || !("default" in loaded) || !isRecord(loaded.default)) {
|
|
throw new Error(`${moduleFile} must default-export defineIntegrationModule(...)`);
|
|
}
|
|
|
|
const directory = path.dirname(moduleFile);
|
|
const module: LoadedIntegrationModule = {
|
|
definition: loaded.default as unknown as IntegrationModuleDefinition,
|
|
directory,
|
|
};
|
|
await validateIntegrationModuleAsync(module);
|
|
if (handAuthored.kinds.has(module.definition.kind)) {
|
|
throw new Error(`Integration kind ${module.definition.kind} exists in both a module and the central registry`);
|
|
}
|
|
if (handAuthored.creatorKinds.has(module.definition.kind)) {
|
|
throw new Error(`Integration kind ${module.definition.kind} exists in both a module and the central creator`);
|
|
}
|
|
if (handAuthored.docsSlugs.has(module.definition.documentation.slug)) {
|
|
throw new Error(
|
|
`Integration documentation slug ${module.definition.documentation.slug} exists in both a module and the central registry`,
|
|
);
|
|
}
|
|
if (kinds.has(module.definition.kind)) throw new Error(`Duplicate integration kind ${module.definition.kind}`);
|
|
kinds.add(module.definition.kind);
|
|
if (docsSlugs.has(module.definition.documentation.slug)) {
|
|
throw new Error(`Duplicate integration documentation slug ${module.definition.documentation.slug}`);
|
|
}
|
|
docsSlugs.add(module.definition.documentation.slug);
|
|
modules.push(module);
|
|
}
|
|
|
|
return modules.toSorted((left, right) => left.definition.kind.localeCompare(right.definition.kind));
|
|
};
|
|
|
|
const renderDefinition = (definition: IntegrationModuleDefinition) => {
|
|
const projected: Record<string, unknown> = {
|
|
name: definition.name,
|
|
secretKinds: definition.secretKinds,
|
|
iconUrl: definition.iconUrl,
|
|
category: definition.categories,
|
|
documentationSlug: definition.documentation.slug,
|
|
};
|
|
if (definition.defaultUrl !== undefined) projected.defaultUrl = definition.defaultUrl;
|
|
if (definition.defaultPort !== undefined) projected.defaultPort = definition.defaultPort;
|
|
if (definition.apiKeySettingsPath !== undefined) projected.apiKeySettingsPath = definition.apiKeySettingsPath;
|
|
if (definition.docker || definition.onboarding) {
|
|
const features: Record<string, unknown> = {};
|
|
if (definition.docker) features.docker = definition.docker;
|
|
if (definition.onboarding) features.onboarding = definition.onboarding;
|
|
projected.features = features;
|
|
}
|
|
return JSON.stringify(projected, null, 2);
|
|
};
|
|
|
|
const renderDefinitionsRegistry = (modules: LoadedIntegrationModule[]) => {
|
|
const entries = modules
|
|
.map((module) => ` ${JSON.stringify(module.definition.kind)}: ${renderDefinition(module.definition)},`)
|
|
.join("\n");
|
|
return `${generatedHeader}export const generatedIntegrationDefinitions = {\n${entries}\n} as const;\n`;
|
|
};
|
|
|
|
const getCreatorImportPath = (module: LoadedIntegrationModule) => {
|
|
const target = path.resolve(module.directory, module.definition.creator.module);
|
|
const relative = toPosixPath(path.relative(generatedRoot, target));
|
|
if (relative.startsWith(".")) return relative;
|
|
return `./${relative}`;
|
|
};
|
|
|
|
const renderCreatorLoader = (module: LoadedIntegrationModule) => {
|
|
const { creator, kind } = module.definition;
|
|
const importPath = getCreatorImportPath(module);
|
|
if (creator.type === "factory") {
|
|
return ` ${JSON.stringify(kind)}: async () => (await import(${JSON.stringify(importPath)})).${creator.exportName},`;
|
|
}
|
|
return ` ${JSON.stringify(kind)}: async () => {\n const IntegrationClass = (await import(${JSON.stringify(importPath)})).${creator.exportName};\n return async (input: IntegrationInput) => new IntegrationClass(input);\n },`;
|
|
};
|
|
|
|
const renderCreatorRegistry = (modules: LoadedIntegrationModule[]) => {
|
|
const entries = modules.map(renderCreatorLoader).join("\n");
|
|
return `${generatedHeader}import type { Integration, IntegrationInput } from "../base/integration";\n\ntype GeneratedIntegrationCreator = (input: IntegrationInput) => Promise<Integration>;\n\nexport const generatedIntegrationCreatorLoaders = {\n${entries}\n} satisfies Record<string, () => Promise<GeneratedIntegrationCreator>>;\n`;
|
|
};
|
|
|
|
const collectFilesAsync = async (directory: string, relativeDirectory = ""): Promise<GeneratedFile[]> => {
|
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
const files: GeneratedFile[] = [];
|
|
for (const entry of entries) {
|
|
const relativePath = path.join(relativeDirectory, entry.name);
|
|
const sourcePath = path.join(directory, entry.name);
|
|
if (entry.isDirectory()) {
|
|
files.push(...(await collectFilesAsync(sourcePath, relativePath)));
|
|
continue;
|
|
}
|
|
if (entry.isFile()) files.push({ path: relativePath, content: await readFile(sourcePath) });
|
|
}
|
|
return files;
|
|
};
|
|
|
|
const createDocsOutputsAsync = async (modules: LoadedIntegrationModule[]) => {
|
|
const outputs: GeneratedFile[] = [];
|
|
for (const module of modules) {
|
|
const source = path.resolve(module.directory, module.definition.documentation.sourceDirectory);
|
|
const target = path.join(docsRoot, module.definition.documentation.slug);
|
|
for (const file of await collectFilesAsync(source)) {
|
|
outputs.push({ path: path.join(target, file.path), content: file.content });
|
|
}
|
|
outputs.push({
|
|
path: path.join(target, generatedDocsMarker),
|
|
content: Buffer.from(`${toPosixPath(path.relative(repositoryRoot, source))}\n`),
|
|
});
|
|
}
|
|
return outputs;
|
|
};
|
|
|
|
const readOrEmptyAsync = async (target: string) => {
|
|
try {
|
|
return await readFile(target);
|
|
} catch {
|
|
return null;
|
|
}
|
|
};
|
|
|
|
const createGeneratedTypeScriptAsync = async (target: string, source: string): Promise<GeneratedFile> => {
|
|
const result = await format(target, source, { printWidth: 120, endOfLine: "lf" });
|
|
const errors = result.errors.filter((error) => error.severity === "Error");
|
|
if (errors.length > 0) {
|
|
throw new Error(`Failed to format ${target}: ${errors.map((error) => error.message).join(", ")}`);
|
|
}
|
|
return { path: target, content: Buffer.from(result.code) };
|
|
};
|
|
|
|
const synchronizeFileAsync = async (file: GeneratedFile) => {
|
|
const current = await readOrEmptyAsync(file.path);
|
|
if (current?.equals(file.content)) return false;
|
|
if (checkOnly) return true;
|
|
await mkdir(path.dirname(file.path), { recursive: true });
|
|
await writeFile(file.path, file.content);
|
|
return true;
|
|
};
|
|
|
|
const removeStaleGeneratedDocsAsync = async (outputs: GeneratedFile[]) => {
|
|
const expected = new Set(outputs.map((output) => path.resolve(output.path)));
|
|
const stale: string[] = [];
|
|
const entries = await readdir(docsRoot, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
if (!entry.isDirectory()) continue;
|
|
const target = path.join(docsRoot, entry.name);
|
|
if (!(await pathExistsAsync(path.join(target, generatedDocsMarker)))) continue;
|
|
for (const file of await collectFilesAsync(target)) {
|
|
const absolutePath = path.resolve(target, file.path);
|
|
if (!expected.has(absolutePath)) stale.push(absolutePath);
|
|
}
|
|
}
|
|
if (!checkOnly) {
|
|
for (const file of stale) await rm(file);
|
|
}
|
|
return stale;
|
|
};
|
|
|
|
const mainAsync = async () => {
|
|
const modules = await loadIntegrationModulesAsync();
|
|
if (modules.length === 0) throw new Error("No vertical integration modules were found");
|
|
|
|
const definitionsTarget = path.join(repositoryRoot, "packages/definitions/src/generated/integration-modules.ts");
|
|
const creatorsTarget = path.join(generatedRoot, "integration-creators.ts");
|
|
const outputs: GeneratedFile[] = [
|
|
await createGeneratedTypeScriptAsync(definitionsTarget, renderDefinitionsRegistry(modules)),
|
|
await createGeneratedTypeScriptAsync(creatorsTarget, renderCreatorRegistry(modules)),
|
|
...(await createDocsOutputsAsync(modules)),
|
|
];
|
|
|
|
const staleDocs = await removeStaleGeneratedDocsAsync(outputs);
|
|
const drifted: string[] = [];
|
|
for (const output of outputs) {
|
|
if (await synchronizeFileAsync(output)) drifted.push(path.relative(repositoryRoot, output.path));
|
|
}
|
|
drifted.push(...staleDocs.map((file) => path.relative(repositoryRoot, file)));
|
|
|
|
if (checkOnly && drifted.length > 0) {
|
|
throw new Error(
|
|
`Integration module artifacts are out of date:\n${drifted.map((file) => ` - ${file}`).join("\n")}\nRun pnpm integrations:sync.`,
|
|
);
|
|
}
|
|
|
|
let action = "synchronized";
|
|
if (checkOnly) action = "verified";
|
|
console.log(`Integration modules ${action}: ${modules.map((module) => module.definition.kind).join(", ")}`);
|
|
};
|
|
|
|
await mainAsync();
|