refactor: streamline widget and integration architecture
This commit is contained in:
ajnart committed
1 parent
ec295fe26e
commit
425cb653a1
243 files changed
+5722
-2255
No files matched your search
@@ -0,0 +1,364 @@
|
||||
import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
import { format } from "oxfmt";
|
||||
|
||||
import type { IntegrationModuleDefinition } from "../../packages/definitions/src/integration-module";
|
||||
|
||||
const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const integrationsRoot = path.join(repositoryRoot, "packages/integrations/src");
|
||||
const generatedRoot = path.join(integrationsRoot, "generated");
|
||||
const docsRoot = path.join(repositoryRoot, "apps/docs/docs/integrations");
|
||||
const integrationDefinitionsSource = path.join(repositoryRoot, "packages/definitions/src/integration.ts");
|
||||
const integrationCreatorsSource = path.join(integrationsRoot, "base/creator.ts");
|
||||
const checkOnly = process.argv.includes("--check");
|
||||
const generatedHeader = "// This file is generated by pnpm integrations:sync. Do not edit it directly.\n\n";
|
||||
const generatedDocsMarker = ".integration-module-generated";
|
||||
|
||||
interface LoadedIntegrationModule {
|
||||
definition: IntegrationModuleDefinition;
|
||||
directory: string;
|
||||
}
|
||||
|
||||
interface GeneratedFile {
|
||||
path: string;
|
||||
content: Buffer;
|
||||
}
|
||||
|
||||
const toPosixPath = (value: string) => value.split(path.sep).join("/");
|
||||
|
||||
const pathExistsAsync = async (target: string) => {
|
||||
try {
|
||||
await stat(target);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const assertPathInside = (parent: string, target: string, label: string) => {
|
||||
const relative = path.relative(parent, target);
|
||||
if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) return;
|
||||
throw new Error(`${label} must stay inside ${parent}`);
|
||||
};
|
||||
|
||||
const findSourceEntryAsync = async (target: string) => {
|
||||
const candidates = [
|
||||
target,
|
||||
`${target}.ts`,
|
||||
`${target}.tsx`,
|
||||
path.join(target, "index.ts"),
|
||||
path.join(target, "index.tsx"),
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (await pathExistsAsync(candidate)) return candidate;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const findIntegrationModuleFilesAsync = async (directory: string): Promise<string[]> => {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
const files: string[] = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.name === "generated" || entry.name === "docs" || entry.name.startsWith(".")) continue;
|
||||
const target = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await findIntegrationModuleFilesAsync(target)));
|
||||
continue;
|
||||
}
|
||||
if (entry.isFile() && entry.name === "module.ts") files.push(target);
|
||||
}
|
||||
|
||||
return files;
|
||||
};
|
||||
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
|
||||
const assertNonEmptyString = (value: unknown, label: string): asserts value is string => {
|
||||
if (typeof value !== "string" || value.trim() === "") throw new Error(`${label} must be a non-empty string`);
|
||||
};
|
||||
|
||||
const escapeRegExp = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
|
||||
const validateExportAsync = async (sourcePath: string, exportName: string, kind: string) => {
|
||||
const source = await readFile(sourcePath, "utf8");
|
||||
const escapedName = escapeRegExp(exportName);
|
||||
const declaration = new RegExp(
|
||||
`export\\s+(?:declare\\s+)?(?:abstract\\s+)?(?:class|const|function)\\s+${escapedName}\\b`,
|
||||
);
|
||||
const exportList = new RegExp(`export\\s*\\{[^}]*\\b${escapedName}\\b[^}]*\\}`, "s");
|
||||
if (declaration.test(source) || exportList.test(source)) return;
|
||||
throw new Error(`Integration ${kind} creator export ${exportName} was not found in ${sourcePath}`);
|
||||
};
|
||||
|
||||
const validateIntegrationModuleAsync = async (module: LoadedIntegrationModule) => {
|
||||
const { definition } = module;
|
||||
assertNonEmptyString(definition.kind, "Integration kind");
|
||||
if (!/^[A-Za-z][A-Za-z0-9]*$/.test(definition.kind)) {
|
||||
throw new Error(`Integration kind ${definition.kind} contains unsupported characters`);
|
||||
}
|
||||
assertNonEmptyString(definition.name, `Integration ${definition.kind} name`);
|
||||
assertNonEmptyString(definition.iconUrl, `Integration ${definition.kind} icon URL`);
|
||||
if (!URL.canParse(definition.iconUrl)) throw new Error(`Integration ${definition.kind} icon URL is invalid`);
|
||||
if (!Array.isArray(definition.secretKinds) || definition.secretKinds.length === 0) {
|
||||
throw new Error(`Integration ${definition.kind} must declare at least one secret alternative`);
|
||||
}
|
||||
if (!Array.isArray(definition.categories) || definition.categories.length === 0) {
|
||||
throw new Error(`Integration ${definition.kind} must declare at least one category`);
|
||||
}
|
||||
if (!/^[a-z0-9][a-z0-9-]*$/.test(definition.documentation.slug)) {
|
||||
throw new Error(`Integration ${definition.kind} has invalid documentation slug ${definition.documentation.slug}`);
|
||||
}
|
||||
if (
|
||||
definition.defaultPort !== undefined &&
|
||||
(!Number.isSafeInteger(definition.defaultPort) || definition.defaultPort < 1)
|
||||
) {
|
||||
throw new Error(`Integration ${definition.kind} default port must be a positive integer`);
|
||||
}
|
||||
|
||||
const docsDirectory = path.resolve(module.directory, definition.documentation.sourceDirectory);
|
||||
assertPathInside(module.directory, docsDirectory, `Integration ${definition.kind} documentation`);
|
||||
if (!(await pathExistsAsync(path.join(docsDirectory, "index.mdx")))) {
|
||||
throw new Error(`Integration ${definition.kind} documentation must contain index.mdx: ${docsDirectory}`);
|
||||
}
|
||||
if (!(await pathExistsAsync(path.join(docsDirectory, "index.ts")))) {
|
||||
throw new Error(`Integration ${definition.kind} documentation must contain index.ts: ${docsDirectory}`);
|
||||
}
|
||||
|
||||
const creatorTarget = path.resolve(module.directory, definition.creator.module);
|
||||
assertPathInside(module.directory, creatorTarget, `Integration ${definition.kind} creator`);
|
||||
const creatorSource = await findSourceEntryAsync(creatorTarget);
|
||||
if (!creatorSource) {
|
||||
throw new Error(`Integration ${definition.kind} creator module does not exist: ${definition.creator.module}`);
|
||||
}
|
||||
assertNonEmptyString(definition.creator.exportName, `Integration ${definition.kind} creator export`);
|
||||
await validateExportAsync(creatorSource, definition.creator.exportName, definition.kind);
|
||||
};
|
||||
|
||||
const loadHandAuthoredRegistryEntriesAsync = async () => {
|
||||
const definitionsSource = await readFile(integrationDefinitionsSource, "utf8");
|
||||
const startMarker = "createIntegrationDefinitions({";
|
||||
const endMarker = "} as const satisfies Record<string, integrationDefinition>);";
|
||||
const start = definitionsSource.indexOf(startMarker);
|
||||
const end = definitionsSource.indexOf(endMarker, start);
|
||||
if (start < 0 || end < 0) throw new Error("Unable to locate the hand-authored integration registry");
|
||||
|
||||
const registry = definitionsSource.slice(start + startMarker.length, end);
|
||||
const kinds = new Set([...registry.matchAll(/^ ([A-Za-z][A-Za-z0-9]*): \{/gm)].map((match) => match[1]));
|
||||
const docsSlugs = new Set(
|
||||
[...registry.matchAll(/^ documentationSlug: "([a-z0-9][a-z0-9-]*)",/gm)].map((match) => match[1]),
|
||||
);
|
||||
|
||||
const creatorsSource = await readFile(integrationCreatorsSource, "utf8");
|
||||
const creatorStartMarker = "const integrationCreatorLoaders = {";
|
||||
const creatorEndMarker = "\n};\n\ntype AssertIntegrationCreatorLoaders";
|
||||
const creatorStart = creatorsSource.indexOf(creatorStartMarker);
|
||||
const creatorEnd = creatorsSource.indexOf(creatorEndMarker, creatorStart);
|
||||
if (creatorStart < 0 || creatorEnd < 0) throw new Error("Unable to locate the hand-authored creator registry");
|
||||
const creatorRegistry = creatorsSource.slice(creatorStart + creatorStartMarker.length, creatorEnd);
|
||||
const creatorKinds = new Set(
|
||||
[...creatorRegistry.matchAll(/^ ([A-Za-z][A-Za-z0-9]*): async /gm)].map((match) => match[1]),
|
||||
);
|
||||
return { kinds, docsSlugs, creatorKinds };
|
||||
};
|
||||
|
||||
const loadIntegrationModulesAsync = async () => {
|
||||
const moduleFiles = (await findIntegrationModuleFilesAsync(integrationsRoot)).toSorted();
|
||||
const handAuthored = await loadHandAuthoredRegistryEntriesAsync();
|
||||
const modules: LoadedIntegrationModule[] = [];
|
||||
const kinds = new Set<string>();
|
||||
const docsSlugs = new Set<string>();
|
||||
|
||||
for (const moduleFile of moduleFiles) {
|
||||
const loaded: unknown = await import(`${pathToFileURL(moduleFile).href}?sync=${Date.now()}`);
|
||||
if (!isRecord(loaded) || !("default" in loaded) || !isRecord(loaded.default)) {
|
||||
throw new Error(`${moduleFile} must default-export defineIntegrationModule(...)`);
|
||||
}
|
||||
|
||||
const directory = path.dirname(moduleFile);
|
||||
const module: LoadedIntegrationModule = {
|
||||
definition: loaded.default as unknown as IntegrationModuleDefinition,
|
||||
directory,
|
||||
};
|
||||
await validateIntegrationModuleAsync(module);
|
||||
if (handAuthored.kinds.has(module.definition.kind)) {
|
||||
throw new Error(`Integration kind ${module.definition.kind} exists in both a module and the central registry`);
|
||||
}
|
||||
if (handAuthored.creatorKinds.has(module.definition.kind)) {
|
||||
throw new Error(`Integration kind ${module.definition.kind} exists in both a module and the central creator`);
|
||||
}
|
||||
if (handAuthored.docsSlugs.has(module.definition.documentation.slug)) {
|
||||
throw new Error(
|
||||
`Integration documentation slug ${module.definition.documentation.slug} exists in both a module and the central registry`,
|
||||
);
|
||||
}
|
||||
if (kinds.has(module.definition.kind)) throw new Error(`Duplicate integration kind ${module.definition.kind}`);
|
||||
kinds.add(module.definition.kind);
|
||||
if (docsSlugs.has(module.definition.documentation.slug)) {
|
||||
throw new Error(`Duplicate integration documentation slug ${module.definition.documentation.slug}`);
|
||||
}
|
||||
docsSlugs.add(module.definition.documentation.slug);
|
||||
modules.push(module);
|
||||
}
|
||||
|
||||
return modules.toSorted((left, right) => left.definition.kind.localeCompare(right.definition.kind));
|
||||
};
|
||||
|
||||
const renderDefinition = (definition: IntegrationModuleDefinition) => {
|
||||
const projected: Record<string, unknown> = {
|
||||
name: definition.name,
|
||||
secretKinds: definition.secretKinds,
|
||||
iconUrl: definition.iconUrl,
|
||||
category: definition.categories,
|
||||
documentationSlug: definition.documentation.slug,
|
||||
};
|
||||
if (definition.defaultUrl !== undefined) projected.defaultUrl = definition.defaultUrl;
|
||||
if (definition.defaultPort !== undefined) projected.defaultPort = definition.defaultPort;
|
||||
if (definition.apiKeySettingsPath !== undefined) projected.apiKeySettingsPath = definition.apiKeySettingsPath;
|
||||
if (definition.docker || definition.onboarding) {
|
||||
const features: Record<string, unknown> = {};
|
||||
if (definition.docker) features.docker = definition.docker;
|
||||
if (definition.onboarding) features.onboarding = definition.onboarding;
|
||||
projected.features = features;
|
||||
}
|
||||
return JSON.stringify(projected, null, 2);
|
||||
};
|
||||
|
||||
const renderDefinitionsRegistry = (modules: LoadedIntegrationModule[]) => {
|
||||
const entries = modules
|
||||
.map((module) => ` ${JSON.stringify(module.definition.kind)}: ${renderDefinition(module.definition)},`)
|
||||
.join("\n");
|
||||
return `${generatedHeader}export const generatedIntegrationDefinitions = {\n${entries}\n} as const;\n`;
|
||||
};
|
||||
|
||||
const getCreatorImportPath = (module: LoadedIntegrationModule) => {
|
||||
const target = path.resolve(module.directory, module.definition.creator.module);
|
||||
const relative = toPosixPath(path.relative(generatedRoot, target));
|
||||
if (relative.startsWith(".")) return relative;
|
||||
return `./${relative}`;
|
||||
};
|
||||
|
||||
const renderCreatorLoader = (module: LoadedIntegrationModule) => {
|
||||
const { creator, kind } = module.definition;
|
||||
const importPath = getCreatorImportPath(module);
|
||||
if (creator.type === "factory") {
|
||||
return ` ${JSON.stringify(kind)}: async () => (await import(${JSON.stringify(importPath)})).${creator.exportName},`;
|
||||
}
|
||||
return ` ${JSON.stringify(kind)}: async () => {\n const IntegrationClass = (await import(${JSON.stringify(importPath)})).${creator.exportName};\n return async (input: IntegrationInput) => new IntegrationClass(input);\n },`;
|
||||
};
|
||||
|
||||
const renderCreatorRegistry = (modules: LoadedIntegrationModule[]) => {
|
||||
const entries = modules.map(renderCreatorLoader).join("\n");
|
||||
return `${generatedHeader}import type { Integration, IntegrationInput } from "../base/integration";\n\ntype GeneratedIntegrationCreator = (input: IntegrationInput) => Promise<Integration>;\n\nexport const generatedIntegrationCreatorLoaders = {\n${entries}\n} satisfies Record<string, () => Promise<GeneratedIntegrationCreator>>;\n`;
|
||||
};
|
||||
|
||||
const collectFilesAsync = async (directory: string, relativeDirectory = ""): Promise<GeneratedFile[]> => {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
const files: GeneratedFile[] = [];
|
||||
for (const entry of entries) {
|
||||
const relativePath = path.join(relativeDirectory, entry.name);
|
||||
const sourcePath = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await collectFilesAsync(sourcePath, relativePath)));
|
||||
continue;
|
||||
}
|
||||
if (entry.isFile()) files.push({ path: relativePath, content: await readFile(sourcePath) });
|
||||
}
|
||||
return files;
|
||||
};
|
||||
|
||||
const createDocsOutputsAsync = async (modules: LoadedIntegrationModule[]) => {
|
||||
const outputs: GeneratedFile[] = [];
|
||||
for (const module of modules) {
|
||||
const source = path.resolve(module.directory, module.definition.documentation.sourceDirectory);
|
||||
const target = path.join(docsRoot, module.definition.documentation.slug);
|
||||
for (const file of await collectFilesAsync(source)) {
|
||||
outputs.push({ path: path.join(target, file.path), content: file.content });
|
||||
}
|
||||
outputs.push({
|
||||
path: path.join(target, generatedDocsMarker),
|
||||
content: Buffer.from(`${toPosixPath(path.relative(repositoryRoot, source))}\n`),
|
||||
});
|
||||
}
|
||||
return outputs;
|
||||
};
|
||||
|
||||
const readOrEmptyAsync = async (target: string) => {
|
||||
try {
|
||||
return await readFile(target);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const createGeneratedTypeScriptAsync = async (target: string, source: string): Promise<GeneratedFile> => {
|
||||
const result = await format(target, source, { printWidth: 120, endOfLine: "lf" });
|
||||
const errors = result.errors.filter((error) => error.severity === "Error");
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`Failed to format ${target}: ${errors.map((error) => error.message).join(", ")}`);
|
||||
}
|
||||
return { path: target, content: Buffer.from(result.code) };
|
||||
};
|
||||
|
||||
const synchronizeFileAsync = async (file: GeneratedFile) => {
|
||||
const current = await readOrEmptyAsync(file.path);
|
||||
if (current?.equals(file.content)) return false;
|
||||
if (checkOnly) return true;
|
||||
await mkdir(path.dirname(file.path), { recursive: true });
|
||||
await writeFile(file.path, file.content);
|
||||
return true;
|
||||
};
|
||||
|
||||
const removeStaleGeneratedDocsAsync = async (outputs: GeneratedFile[]) => {
|
||||
const expected = new Set(outputs.map((output) => path.resolve(output.path)));
|
||||
const stale: string[] = [];
|
||||
const entries = await readdir(docsRoot, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const target = path.join(docsRoot, entry.name);
|
||||
if (!(await pathExistsAsync(path.join(target, generatedDocsMarker)))) continue;
|
||||
for (const file of await collectFilesAsync(target)) {
|
||||
const absolutePath = path.resolve(target, file.path);
|
||||
if (!expected.has(absolutePath)) stale.push(absolutePath);
|
||||
}
|
||||
}
|
||||
if (!checkOnly) {
|
||||
for (const file of stale) await rm(file);
|
||||
}
|
||||
return stale;
|
||||
};
|
||||
|
||||
const mainAsync = async () => {
|
||||
const modules = await loadIntegrationModulesAsync();
|
||||
if (modules.length === 0) throw new Error("No vertical integration modules were found");
|
||||
|
||||
const definitionsTarget = path.join(repositoryRoot, "packages/definitions/src/generated/integration-modules.ts");
|
||||
const creatorsTarget = path.join(generatedRoot, "integration-creators.ts");
|
||||
const outputs: GeneratedFile[] = [
|
||||
await createGeneratedTypeScriptAsync(definitionsTarget, renderDefinitionsRegistry(modules)),
|
||||
await createGeneratedTypeScriptAsync(creatorsTarget, renderCreatorRegistry(modules)),
|
||||
...(await createDocsOutputsAsync(modules)),
|
||||
];
|
||||
|
||||
const staleDocs = await removeStaleGeneratedDocsAsync(outputs);
|
||||
const drifted: string[] = [];
|
||||
for (const output of outputs) {
|
||||
if (await synchronizeFileAsync(output)) drifted.push(path.relative(repositoryRoot, output.path));
|
||||
}
|
||||
drifted.push(...staleDocs.map((file) => path.relative(repositoryRoot, file)));
|
||||
|
||||
if (checkOnly && drifted.length > 0) {
|
||||
throw new Error(
|
||||
`Integration module artifacts are out of date:\n${drifted.map((file) => ` - ${file}`).join("\n")}\nRun pnpm integrations:sync.`,
|
||||
);
|
||||
}
|
||||
|
||||
let action = "synchronized";
|
||||
if (checkOnly) action = "verified";
|
||||
console.log(`Integration modules ${action}: ${modules.map((module) => module.definition.kind).join(", ")}`);
|
||||
};
|
||||
|
||||
await mainAsync();
|
||||
Reference in new issue
Block a user