Files
homarr/scripts/widgets/sync-widget-modules.mts
T

476 lines
22 KiB
TypeScript

import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { format } from "oxfmt";
import { getIntegrationKindsByCategory } from "../../packages/definitions/src/integration";
import type {
WidgetModuleDefinition,
WidgetModuleIntegrationCapability,
WidgetModuleOperation,
WidgetModuleRouterRegistration,
} from "../../packages/definitions/src/widget-module";
const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const widgetsRoot = path.join(repositoryRoot, "packages/widgets/src");
const apiRoutersRoot = path.join(repositoryRoot, "packages/api/src/router/widgets");
const checkOnly = process.argv.includes("--check");
const generatedHeader = "// This file is generated by pnpm widgets:sync. Do not edit it directly.\n\n";
const generatedDocsMarker = ".widget-module-generated";
interface LoadedWidgetModule {
definition: WidgetModuleDefinition;
directory: string;
relativeDirectory: string;
}
interface GeneratedFile {
path: string;
content: Buffer;
}
interface GroupedRouter extends WidgetModuleRouterRegistration {
widgetKinds: string[];
}
const toPosixPath = (value: string) => value.split(path.sep).join("/");
const quote = (value: string) => JSON.stringify(value);
const indent = (value: string, spaces: number) => {
const prefix = " ".repeat(spaces);
return value
.split("\n")
.map((line) => `${prefix}${line}`)
.join("\n");
};
const pathExists = async (target: string) => {
try {
await stat(target);
return true;
} catch {
return false;
}
};
const assertPathInside = (parent: string, target: string, label: string) => {
const relative = path.relative(parent, target);
if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) return;
throw new Error(`${label} must stay inside ${parent}`);
};
const findSourceEntryAsync = async (target: string) => {
const candidates = [
target,
`${target}.ts`,
`${target}.tsx`,
path.join(target, "index.ts"),
path.join(target, "index.tsx"),
];
for (const candidate of candidates) {
if (await pathExists(candidate)) return candidate;
}
return null;
};
const findWidgetModuleFilesAsync = async (directory: string): Promise<string[]> => {
const entries = await readdir(directory, { withFileTypes: true });
const files: string[] = [];
for (const entry of entries) {
if (entry.name === "generated" || entry.name === "docs" || entry.name.startsWith(".")) continue;
const target = path.join(directory, entry.name);
if (entry.isDirectory()) {
files.push(...(await findWidgetModuleFilesAsync(target)));
continue;
}
if (entry.isFile() && entry.name === "module.ts") files.push(target);
}
return files;
};
const isRecord = (value: unknown): value is Record<string, unknown> =>
value !== null && typeof value === "object" && !Array.isArray(value);
const assertNonEmptyString = (value: unknown, label: string): asserts value is string => {
if (typeof value !== "string" || value.trim() === "") throw new Error(`${label} must be a non-empty string`);
};
const validateOperation = (module: LoadedWidgetModule, operation: WidgetModuleOperation, names: Set<string>) => {
assertNonEmptyString(operation.name, `${module.definition.kind} operation name`);
if (names.has(operation.name)) {
throw new Error(`Widget ${module.definition.kind} declares operation ${operation.name} more than once`);
}
names.add(operation.name);
if (!Array.isArray(operation.path) || operation.path.length === 0) {
throw new Error(`Widget ${module.definition.kind} operation ${operation.name} must declare a non-empty path`);
}
for (const part of operation.path) {
assertNonEmptyString(part, `${module.definition.kind}.${operation.name} path segment`);
}
if (operation.client?.staleTimeSeconds !== undefined && operation.client.refetchIntervalSeconds === undefined) {
throw new Error(
`Widget ${module.definition.kind} operation ${operation.name} must declare a refetch interval with stale time`,
);
}
const cache = operation.serverCache;
if (!cache) return;
if (!/^[a-z0-9][a-z0-9:-]*$/.test(cache.namespace)) {
throw new Error(`Widget ${module.definition.kind} has invalid cache namespace ${cache.namespace}`);
}
if (!Number.isSafeInteger(cache.ttlMs) || cache.ttlMs < 0) {
throw new Error(`Widget ${module.definition.kind} cache TTL must be a non-negative integer`);
}
};
const validateWidgetModuleAsync = async (module: LoadedWidgetModule) => {
const { definition } = module;
if ("enabled" in definition || "disabled" in definition) {
throw new Error(`Widget ${definition.kind} must not declare a feature availability flag`);
}
assertNonEmptyString(definition.kind, "Widget kind");
if (!/^[A-Za-z][A-Za-z0-9-]*$/.test(definition.kind)) {
throw new Error(`Widget kind ${definition.kind} contains unsupported characters`);
}
if (!/^Icon[A-Za-z0-9]+$/.test(definition.icon)) {
throw new Error(`Widget ${definition.kind} has invalid Tabler icon export ${definition.icon}`);
}
if (!/^[a-z0-9][a-z0-9-]*$/.test(definition.documentation.slug)) {
throw new Error(`Widget ${definition.kind} has invalid documentation slug ${definition.documentation.slug}`);
}
const clientTarget = path.resolve(module.directory, definition.clientEntry);
assertPathInside(module.directory, clientTarget, `Widget ${definition.kind} client entry`);
if (!(await findSourceEntryAsync(clientTarget))) {
throw new Error(`Widget ${definition.kind} client entry does not exist: ${definition.clientEntry}`);
}
const docsDirectory = path.resolve(module.directory, definition.documentation.sourceDirectory);
assertPathInside(module.directory, docsDirectory, `Widget ${definition.kind} documentation`);
if (!(await pathExists(path.join(docsDirectory, "index.mdx")))) {
throw new Error(`Widget ${definition.kind} documentation must contain index.mdx: ${docsDirectory}`);
}
const operationNames = new Set<string>();
for (const operation of definition.operations ?? []) validateOperation(module, operation, operationNames);
for (const router of definition.routers ?? []) {
assertNonEmptyString(router.namespace, `${definition.kind} router namespace`);
assertNonEmptyString(router.module, `${definition.kind} router module`);
assertNonEmptyString(router.exportName, `${definition.kind} router export`);
const routerTarget = path.resolve(apiRoutersRoot, router.module);
assertPathInside(apiRoutersRoot, routerTarget, `Widget ${definition.kind} router module`);
if (!(await findSourceEntryAsync(routerTarget))) {
throw new Error(`Widget ${definition.kind} router module does not exist: ${router.module}`);
}
}
};
const loadWidgetModulesAsync = async () => {
const moduleFiles = (await findWidgetModuleFilesAsync(widgetsRoot)).toSorted();
const modules: LoadedWidgetModule[] = [];
const kinds = new Set<string>();
const docsSlugs = new Set<string>();
const cacheNamespaces = new Set<string>();
for (const moduleFile of moduleFiles) {
const loaded: unknown = await import(`${pathToFileURL(moduleFile).href}?sync=${Date.now()}`);
if (!isRecord(loaded) || !("default" in loaded) || !isRecord(loaded.default)) {
throw new Error(`${moduleFile} must default-export defineWidgetModule(...)`);
}
const directory = path.dirname(moduleFile);
const module: LoadedWidgetModule = {
definition: loaded.default as unknown as WidgetModuleDefinition,
directory,
relativeDirectory: toPosixPath(path.relative(widgetsRoot, directory)),
};
await validateWidgetModuleAsync(module);
if (kinds.has(module.definition.kind)) throw new Error(`Duplicate widget kind ${module.definition.kind}`);
kinds.add(module.definition.kind);
if (docsSlugs.has(module.definition.documentation.slug)) {
throw new Error(`Duplicate widget documentation slug ${module.definition.documentation.slug}`);
}
docsSlugs.add(module.definition.documentation.slug);
for (const operation of module.definition.operations ?? []) {
const namespace = operation.serverCache?.namespace;
if (!namespace) continue;
if (cacheNamespaces.has(namespace)) throw new Error(`Duplicate widget cache namespace ${namespace}`);
cacheNamespaces.add(namespace);
}
modules.push(module);
}
return modules.toSorted((left, right) => left.definition.kind.localeCompare(right.definition.kind));
};
const resolveIntegrationKinds = (capability: WidgetModuleIntegrationCapability) => {
const kinds = new Set(capability.kinds ?? []);
for (const category of capability.categories ?? []) {
for (const kind of getIntegrationKindsByCategory(category)) kinds.add(kind);
}
for (const excludedKind of capability.excludeKinds ?? []) kinds.delete(excludedKind);
if (kinds.size === 0) throw new Error("A widget integration capability must resolve to at least one kind");
return [...kinds];
};
const renderFeatureDescriptor = (module: LoadedWidgetModule) => {
const { definition } = module;
const fields = [`documentationSlug: ${quote(definition.documentation.slug)}`];
if (definition.defaultSize) {
fields.push(`defaultSize: { width: ${definition.defaultSize.width}, height: ${definition.defaultSize.height} }`);
}
if (definition.displayNameFromIntegration) fields.push("displayNameFromIntegration: true");
if (definition.integration) {
const capability = [`integrations: ${JSON.stringify(resolveIntegrationKinds(definition.integration))} as const`];
if (definition.integration.connectionOptional) capability.push("connectionOptional: true");
if (definition.integration.serverMaxIntegrations !== undefined) {
capability.push(`serverMaxIntegrations: ${definition.integration.serverMaxIntegrations}`);
}
fields.push(`capability: { ${capability.join(", ")} }`);
}
const queryOperations = (definition.operations ?? []).filter((operation) => (operation.kind ?? "query") === "query");
if (queryOperations.length > 0) {
const intervals = new Set(
queryOperations
.map((operation) => operation.client?.refetchIntervalSeconds)
.filter((interval) => interval !== undefined),
);
if (intervals.size > 1) {
throw new Error(`Widget ${definition.kind} has conflicting root refetch intervals; split the router path policy`);
}
const queryFields = [`paths: ${JSON.stringify(queryOperations.map((operation) => operation.path))} as const`];
const interval = intervals.values().next().value;
if (interval !== undefined) queryFields.push(`refetchIntervalSeconds: ${String(interval)}`);
fields.push(`query: { ${queryFields.join(", ")} }`);
}
return `{\n${indent(fields.map((field) => `${field},`).join("\n"), 4)}\n }`;
};
const renderDefinitionsRegistry = (modules: LoadedWidgetModule[]) => {
const catalogEntries = modules
.map((module) => ` ${quote(module.definition.kind)}: ${renderFeatureDescriptor(module)},`)
.join("\n");
const cacheEntries = modules
.filter((module) => (module.definition.operations ?? []).some((operation) => operation.serverCache))
.map((module) => {
const operations = (module.definition.operations ?? [])
.filter((operation) => operation.serverCache)
.map((operation) => ` ${quote(operation.name)}: ${JSON.stringify(operation.serverCache)},`)
.join("\n");
return ` ${quote(module.definition.kind)}: {\n${operations}\n },`;
})
.join("\n");
const operationPolicies = modules.flatMap((module) =>
(module.definition.operations ?? []).map((operation) => ({
widgetKind: module.definition.kind,
operation: operation.name,
kind: operation.kind ?? "query",
path: operation.path,
refetchIntervalSeconds: operation.client?.refetchIntervalSeconds,
staleTimeSeconds: operation.client?.staleTimeSeconds,
persist: operation.client?.persist ?? false,
})),
);
const documentationSources = Object.fromEntries(
modules.map((module) => [
module.definition.documentation.slug,
toPosixPath(
path.join("packages/widgets/src", module.relativeDirectory, module.definition.documentation.sourceDirectory),
),
]),
);
return `${generatedHeader}export const generatedWidgetFeatureCatalog = {\n${catalogEntries}\n} as const;\n\nexport const widgetServerCachePolicies = {\n${cacheEntries}\n} as const;\n\nexport const generatedWidgetOperationPolicies = ${JSON.stringify(operationPolicies, null, 2)} as const;\n\nexport const generatedWidgetDocumentationSources = ${JSON.stringify(documentationSources, null, 2)} as const;\n\nexport const getWidgetServerCachePolicy = <\n TKind extends keyof typeof widgetServerCachePolicies,\n TOperation extends keyof (typeof widgetServerCachePolicies)[TKind],\n>(\n kind: TKind,\n operation: TOperation,\n): (typeof widgetServerCachePolicies)[TKind][TOperation] => widgetServerCachePolicies[kind][operation];\n`;
};
const getClientImportPath = (module: LoadedWidgetModule) => {
const base = `../${module.relativeDirectory}`;
if (module.definition.clientEntry === ".") return base;
return `${base}/${module.definition.clientEntry.replace(/^\.\//, "")}`;
};
const renderWidgetRegistry = (modules: LoadedWidgetModule[]) => {
const typeImports = modules
.map((module, index) => `import type * as widgetModule${index} from ${quote(getClientImportPath(module))};`)
.join("\n");
const loaders = modules
.map((module) => ` ${quote(module.definition.kind)}: () => import(${quote(getClientImportPath(module))}),`)
.join("\n");
const imports = modules
.map((module, index) => ` ${quote(module.definition.kind)}: typeof widgetModule${index};`)
.join("\n");
return `${generatedHeader}${typeImports}\n\nexport const generatedWidgetModuleLoaders = {\n${loaders}\n};\n\nexport interface GeneratedWidgetImports {\n${imports}\n}\n`;
};
const renderIconRegistry = (modules: LoadedWidgetModule[]) => {
const icons = [...new Set(modules.map((module) => module.definition.icon))].toSorted();
const entries = modules.map((module) => ` ${quote(module.definition.kind)}: ${module.definition.icon},`).join("\n");
return `${generatedHeader}import { ${icons.join(", ")} } from "@tabler/icons-react";\nimport type { TablerIcon } from "@tabler/icons-react";\n\nimport type { WidgetKind } from "@homarr/definitions";\n\nexport const generatedWidgetCatalogIcons = {\n${entries}\n} satisfies Partial<Record<WidgetKind, TablerIcon>>;\n`;
};
const renderLocalWidgetProjection = (module: LoadedWidgetModule) =>
`${generatedHeader}export { ${module.definition.icon} as widgetIcon } from "@tabler/icons-react";\n`;
const groupRouters = (modules: LoadedWidgetModule[]) => {
const routers = new Map<string, GroupedRouter>();
for (const module of modules) {
for (const registration of module.definition.routers ?? []) {
const widgetKinds = [...(registration.additionalWidgetKinds ?? []), module.definition.kind];
const existing = routers.get(registration.namespace);
if (!existing) {
routers.set(registration.namespace, { ...registration, widgetKinds });
continue;
}
if (
existing.module !== registration.module ||
existing.exportName !== registration.exportName ||
existing.mcp !== registration.mcp
) {
throw new Error(`Router namespace ${registration.namespace} has conflicting registrations`);
}
existing.widgetKinds = [...new Set([...existing.widgetKinds, ...widgetKinds])];
}
}
return [...routers.values()].toSorted((left, right) => left.namespace.localeCompare(right.namespace));
};
const renderApiRegistry = (modules: LoadedWidgetModule[]) => {
const entries = groupRouters(modules)
.map((router) => {
const mcp = router.mcp === undefined ? "false" : JSON.stringify(router.mcp);
return ` ${quote(router.namespace)}: registerGeneratedWidgetRouter(\n ${JSON.stringify(router.widgetKinds)},\n () => import(${quote(router.module)}).then((module) => module.${router.exportName}),\n ${mcp},\n ),`;
})
.join("\n");
return `${generatedHeader}import type { AnyTRPCRouter } from "@trpc/server";\n\nimport type { WidgetKind } from "@homarr/definitions";\n\nconst registerGeneratedWidgetRouter = <TRouter extends AnyTRPCRouter>(\n widgetKinds: readonly [WidgetKind, ...WidgetKind[]],\n loadRouter: () => Promise<TRouter>,\n mcp: false | true | string,\n) => ({ widgetKinds, loadRouter, mcp });\n\nexport const generatedWidgetRouterRegistry = {\n${entries}\n};\n`;
};
const collectFilesAsync = async (directory: string, relativeDirectory = ""): Promise<GeneratedFile[]> => {
const entries = await readdir(directory, { withFileTypes: true });
const files: GeneratedFile[] = [];
for (const entry of entries) {
const relativePath = path.join(relativeDirectory, entry.name);
const sourcePath = path.join(directory, entry.name);
if (entry.isDirectory()) {
files.push(...(await collectFilesAsync(sourcePath, relativePath)));
continue;
}
if (entry.isFile()) files.push({ path: relativePath, content: await readFile(sourcePath) });
}
return files;
};
const createDocsOutputsAsync = async (modules: LoadedWidgetModule[]) => {
const outputs: GeneratedFile[] = [];
for (const module of modules) {
const source = path.resolve(module.directory, module.definition.documentation.sourceDirectory);
const target = path.join(repositoryRoot, "apps/docs/docs/widgets", module.definition.documentation.slug);
const sourceFiles = await collectFilesAsync(source);
for (const file of sourceFiles) {
outputs.push({ path: path.join(target, file.path), content: file.content });
}
outputs.push({
path: path.join(target, generatedDocsMarker),
content: Buffer.from(`${toPosixPath(path.relative(repositoryRoot, source))}\n`),
});
}
return outputs;
};
const readOrEmptyAsync = async (target: string) => {
try {
return await readFile(target);
} catch {
return null;
}
};
const createGeneratedTypeScriptAsync = async (target: string, source: string): Promise<GeneratedFile> => {
const result = await format(target, source, { printWidth: 120, endOfLine: "lf" });
const errors = result.errors.filter((error) => error.severity === "Error");
if (errors.length > 0) {
throw new Error(`Failed to format ${target}: ${errors.map((error) => error.message).join(", ")}`);
}
return { path: target, content: Buffer.from(result.code) };
};
const synchronizeFileAsync = async (file: GeneratedFile) => {
const current = await readOrEmptyAsync(file.path);
if (current?.equals(file.content)) return false;
if (checkOnly) return true;
await mkdir(path.dirname(file.path), { recursive: true });
await writeFile(file.path, file.content);
return true;
};
const removeStaleGeneratedDocsAsync = async (modules: LoadedWidgetModule[], outputs: GeneratedFile[]) => {
const expected = new Set(outputs.map((output) => path.resolve(output.path)));
const stale: string[] = [];
const docsRoot = path.join(repositoryRoot, "apps/docs/docs/widgets");
const targetDirectories = new Set(modules.map((module) => path.join(docsRoot, module.definition.documentation.slug)));
for (const entry of await readdir(docsRoot, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const target = path.join(docsRoot, entry.name);
if (await pathExists(path.join(target, generatedDocsMarker))) targetDirectories.add(target);
}
for (const target of targetDirectories) {
if (!(await pathExists(path.join(target, generatedDocsMarker)))) continue;
for (const file of await collectFilesAsync(target)) {
const absolutePath = path.resolve(target, file.path);
if (!expected.has(absolutePath)) stale.push(absolutePath);
}
}
if (!checkOnly) {
for (const file of stale) await rm(file);
}
return stale;
};
const mainAsync = async () => {
const modules = await loadWidgetModulesAsync();
if (modules.length === 0) throw new Error("No vertical widget modules were found");
const definitionsTarget = path.join(repositoryRoot, "packages/definitions/src/generated/widget-modules.ts");
const widgetsTarget = path.join(repositoryRoot, "packages/widgets/src/generated/widget-modules.ts");
const iconsTarget = path.join(repositoryRoot, "packages/ui/src/generated/widget-icons.ts");
const apiTarget = path.join(repositoryRoot, "packages/api/src/router/widgets/generated-registry.ts");
const outputs: GeneratedFile[] = [
await createGeneratedTypeScriptAsync(definitionsTarget, renderDefinitionsRegistry(modules)),
await createGeneratedTypeScriptAsync(widgetsTarget, renderWidgetRegistry(modules)),
await createGeneratedTypeScriptAsync(iconsTarget, renderIconRegistry(modules)),
await createGeneratedTypeScriptAsync(apiTarget, renderApiRegistry(modules)),
...(await Promise.all(
modules.map(async (module) => {
const target = path.join(module.directory, "module.generated.ts");
return await createGeneratedTypeScriptAsync(target, renderLocalWidgetProjection(module));
}),
)),
...(await createDocsOutputsAsync(modules)),
];
const staleDocs = await removeStaleGeneratedDocsAsync(modules, outputs);
const drifted: string[] = [];
for (const output of outputs) {
if (await synchronizeFileAsync(output)) drifted.push(path.relative(repositoryRoot, output.path));
}
drifted.push(...staleDocs.map((file) => path.relative(repositoryRoot, file)));
if (checkOnly && drifted.length > 0) {
throw new Error(
`Widget module artifacts are out of date:\n${drifted.map((file) => ` - ${file}`).join("\n")}\nRun pnpm widgets:sync.`,
);
}
const action = checkOnly ? "verified" : "synchronized";
console.log(`Widget modules ${action}: ${modules.map((module) => module.definition.kind).join(", ")}`);
};
await mainAsync();