docs: improve Fumadocs search and AI integration
This commit is contained in:
ajnart committed
1 parent
6704ef2c13
commit
16ba3c6977
33 files changed
+1354
-280
No files matched your search
@@ -0,0 +1,45 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "homarr-docs-links-"));
|
||||
const validator = fileURLToPath(new URL("./validate-links.ts", import.meta.url));
|
||||
|
||||
try {
|
||||
await mkdir(path.join(directory, "docs"));
|
||||
await mkdir(path.join(directory, "out/docs/child"), { recursive: true });
|
||||
await writeFile(path.join(directory, "out/docs/child/index.html"), '<h1 id="child">Child</h1>');
|
||||
await writeFile(path.join(directory, "out/asset.svg"), "<svg />");
|
||||
|
||||
const cases = [
|
||||
{ name: "valid anchors, relative routes, assets and external URLs", href: "#section", expected: 0 },
|
||||
{ name: "same-page generated anchor", href: "#missing", expected: 1 },
|
||||
{ name: "cross-page anchor", href: "/docs/child/#missing", expected: 1 },
|
||||
{ name: "rendered route", href: "/missing", expected: 1 },
|
||||
{ name: "same-origin absolute route", href: "https://homarr.dev/missing", expected: 1 },
|
||||
{ name: "rendered asset", href: "/missing.svg", expected: 1 },
|
||||
{ name: "source-only link", href: "#section", source: "[Missing](/missing)", expected: 1 },
|
||||
{ name: "source-only anchor", href: "#section", source: "[Missing](/docs/#missing)", expected: 1 },
|
||||
];
|
||||
|
||||
for (const test of cases) {
|
||||
await writeFile(path.join(directory, "docs/index.mdx"), test.source ?? "[Valid](/docs/#section)");
|
||||
await writeFile(
|
||||
path.join(directory, "out/docs/index.html"),
|
||||
`<main><h1 id="section">Section</h1><a href="${test.href}">Test</a>
|
||||
<a href="child/#child">Child</a><a href="/asset.svg">Asset</a>
|
||||
<a href="https://example.com/missing">External</a><a href="#top">Top</a></main>`,
|
||||
);
|
||||
const result = spawnSync(process.execPath, ["--import", import.meta.resolve("tsx"), validator], {
|
||||
cwd: directory,
|
||||
encoding: "utf8",
|
||||
});
|
||||
assert.equal(result.status, test.expected, `${test.name}\n${result.stdout}\n${result.stderr}`);
|
||||
}
|
||||
console.log(`${cases.length} link validation checks passed`);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { load } from "cheerio";
|
||||
import { printErrors, readFiles, validateFiles, type ScanResult } from "next-validate-link";
|
||||
import { printErrors, readFiles, validateFiles, type ScanResult, type UrlMeta } from "next-validate-link";
|
||||
|
||||
const outputDirectory = path.resolve("out");
|
||||
|
||||
@@ -11,7 +11,7 @@ const toContentUrl = (file: string) => {
|
||||
if (normalized.startsWith("docs/")) {
|
||||
const slug = normalized
|
||||
.slice("docs/".length)
|
||||
.replace(/\/(index)?\.mdx?$/, "")
|
||||
.replace(/(?:^|\/)index\.mdx?$/, "")
|
||||
.replace(/\.mdx?$/, "");
|
||||
return `/docs${slug ? `/${slug}` : ""}/`;
|
||||
}
|
||||
@@ -25,8 +25,8 @@ const toContentUrl = (file: string) => {
|
||||
|
||||
const files = await readFiles(["docs/**/*.{md,mdx}", "blog/**/*.{md,mdx}"], { pathToUrl: toContentUrl });
|
||||
const output = await readdir(outputDirectory, { recursive: true, withFileTypes: true });
|
||||
const urls = new Map<string, object>();
|
||||
const renderedPages: { file: string; url: string }[] = [];
|
||||
const urls = new Map<string, UrlMeta>();
|
||||
const renderedPages: { url: string; links: string[] }[] = [];
|
||||
|
||||
for (const entry of output) {
|
||||
if (!entry.isFile()) continue;
|
||||
@@ -37,9 +37,20 @@ for (const entry of output) {
|
||||
continue;
|
||||
}
|
||||
const url = `/${relative.replace(/\/?index\.html$/, "")}`.replace(/\/$/, "") || "/";
|
||||
urls.set(url, {});
|
||||
urls.set(`${url}/`, {});
|
||||
renderedPages.push({ file, url });
|
||||
const $ = load(await readFile(file, "utf8"));
|
||||
const hashes = $("[id], a[name]")
|
||||
.toArray()
|
||||
.flatMap((element) => [$(element).attr("id"), $(element).attr("name")].filter((value) => value !== undefined));
|
||||
const meta = { hashes };
|
||||
urls.set(url, meta);
|
||||
urls.set(url === "/" ? url : `${url}/`, meta);
|
||||
renderedPages.push({
|
||||
url,
|
||||
links: $("a[href]")
|
||||
.toArray()
|
||||
.map((element) => $(element).attr("href"))
|
||||
.filter((href) => href !== undefined),
|
||||
});
|
||||
}
|
||||
|
||||
const scanned: ScanResult = { urls, fallbackUrls: [] };
|
||||
@@ -47,38 +58,42 @@ const results = await validateFiles(files, {
|
||||
scanned,
|
||||
checkExternal: false,
|
||||
checkRelativePaths: "exists",
|
||||
ignoreFragment: true,
|
||||
markdown: {
|
||||
components: {
|
||||
Link: { attributes: ["href", "to"] },
|
||||
},
|
||||
},
|
||||
whitelist: (url) =>
|
||||
/^(?:https?:|mailto:|tel:|#)/.test(url) ||
|
||||
/^\/(?:api\/|custom-widgets\/|data\/|img\/|workshop(?:\/|$)|llms(?:-|\.|\/)|robots\.txt|sitemap\.xml)/.test(url),
|
||||
whitelist: (url) => /^(?:https?:|mailto:|tel:)/.test(url),
|
||||
});
|
||||
|
||||
printErrors(
|
||||
results.filter((result) => result.errors.length > 0),
|
||||
true,
|
||||
);
|
||||
const sourceErrors = results.filter((result) => result.errors.length > 0);
|
||||
printErrors(sourceErrors);
|
||||
if (sourceErrors.length > 0) process.exitCode = 1;
|
||||
|
||||
const renderedLinkErrors = new Set<string>();
|
||||
|
||||
for (const page of renderedPages) {
|
||||
const $ = load(await readFile(page.file, "utf8"));
|
||||
|
||||
$("main a[href]").each((_, element) => {
|
||||
const href = $(element).attr("href");
|
||||
if (!href || /^(?:https?:|mailto:|tel:|#)/.test(href)) return;
|
||||
|
||||
const target = new URL(href, `https://homarr.dev${page.url}`).pathname;
|
||||
if (/^\/(?:_next\/|api\/|custom-widgets\/|data\/|img\/|videos\/)/.test(target)) return;
|
||||
if (/\.(?:avif|gif|jpe?g|json|md|mp4|png|svg|webm|webp)$/i.test(target)) return;
|
||||
if (urls.has(target) || urls.has(target.replace(/\/$/, "")) || urls.has(`${target}/`)) return;
|
||||
|
||||
renderedLinkErrors.add(`${page.url}: ${href} resolves to missing route ${target}`);
|
||||
});
|
||||
for (const href of page.links) {
|
||||
if (!href) continue;
|
||||
try {
|
||||
// The export serves directories with trailing slashes, including relative links.
|
||||
const target = new URL(href, `https://homarr.dev${page.url.replace(/\/$/, "")}/`);
|
||||
if (target.origin !== "https://homarr.dev") continue;
|
||||
const pathname = decodeURI(target.pathname);
|
||||
const meta = urls.get(pathname) ?? urls.get(pathname.replace(/\/$/, "")) ?? urls.get(`${pathname}/`);
|
||||
if (!meta) {
|
||||
renderedLinkErrors.add(`${page.url}: ${href} resolves to missing route ${pathname}`);
|
||||
continue;
|
||||
}
|
||||
// Text fragments do not refer to element IDs; validate any preceding anchor.
|
||||
const hash = decodeURIComponent(target.hash.slice(1).split(":~:text=")[0]);
|
||||
if (hash && hash !== "top" && meta.hashes && !meta.hashes.includes(hash)) {
|
||||
renderedLinkErrors.add(`${page.url}: ${href} resolves to missing anchor #${hash} on ${pathname}`);
|
||||
}
|
||||
} catch {
|
||||
renderedLinkErrors.add(`${page.url}: invalid URL ${href}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (renderedLinkErrors.size > 0) {
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { access, readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { load } from "cheerio";
|
||||
import { staticClient } from "fumadocs-core/search/client/orama-static";
|
||||
|
||||
const outputDirectory = path.resolve("out");
|
||||
const index = await readFile(path.join(outputDirectory, "api/search"), "utf8");
|
||||
// Exercise the same client and exported database as the browser, without a server.
|
||||
const from = `data:application/json;base64,${Buffer.from(index).toString("base64")}`;
|
||||
const client = staticClient({ from });
|
||||
const cases = [
|
||||
{ query: "Sonarr", type: "page", url: "/docs/integrations/sonarr" },
|
||||
{ query: "Docker Compose", type: "heading", url: "/docs/getting-started/installation/docker#docker-compose" },
|
||||
{ query: "SECRET_ENCRYPTION_KEY", type: "text", url: "/docs/advanced/environment-variables" },
|
||||
] as const;
|
||||
const verifiedUrls = new Set<string>();
|
||||
|
||||
for (const test of cases) {
|
||||
const results = await client.search(test.query);
|
||||
assert(
|
||||
results.some(
|
||||
(result) =>
|
||||
result.type === test.type &&
|
||||
result.url.split("#")[0] === test.url.split("#")[0] &&
|
||||
(!test.url.includes("#") || result.url === test.url),
|
||||
),
|
||||
`${test.query}: missing ${test.type} result for ${test.url}`,
|
||||
);
|
||||
|
||||
for (const result of results) {
|
||||
assert(result.url.startsWith("/docs/"), `Unexpected search destination: ${result.url}`);
|
||||
if (verifiedUrls.has(result.url)) continue;
|
||||
const url = new URL(result.url, "https://homarr.dev");
|
||||
const html = await readFile(path.join(outputDirectory, decodeURIComponent(url.pathname), "index.html"), "utf8");
|
||||
if (url.hash) {
|
||||
const id = decodeURIComponent(url.hash.slice(1));
|
||||
assert(
|
||||
load(html)("[id]")
|
||||
.toArray()
|
||||
.some((element) => element.attribs.id === id),
|
||||
`Missing search anchor: ${result.url}`,
|
||||
);
|
||||
}
|
||||
verifiedUrls.add(result.url);
|
||||
}
|
||||
console.log(`${test.query}: ${results.length} results; expected ${test.type} found`);
|
||||
}
|
||||
|
||||
assert.deepEqual(await client.search("homarrsearchnomatchqzxv987654321"), [], "Unknown words should return no results");
|
||||
const credentialResults = await staticClient({ from, search: { limit: 1000 } }).search("API Key");
|
||||
assert(
|
||||
credentialResults.some((result) => result.type === "text" && result.url.startsWith("/docs/integrations/sonarr#")),
|
||||
"Sonarr credential metadata must be searchable",
|
||||
);
|
||||
|
||||
for (const [slug, expected] of [
|
||||
["integrations/sonarr", ["API Key", "Management", "Calendar"]],
|
||||
["widgets/calendar", ["Radarr release type", "Default:", "Show unmonitored"]],
|
||||
] as const) {
|
||||
const markdown = await readFile(path.join(outputDirectory, "llms.mdx/docs", slug, "content.md"), "utf8");
|
||||
for (const text of expected) assert(markdown.includes(text), `${slug}: missing resolved metadata ${text}`);
|
||||
assert(!/<(?:Integration|Widget|Adding)[A-Z][^>]*[/>]/.test(markdown), `${slug}: unresolved metadata component`);
|
||||
}
|
||||
|
||||
const discovery = await readFile(path.join(outputDirectory, "llms.txt"), "utf8");
|
||||
const markdownLinks = [...discovery.matchAll(/\]\(([^)]*\/llms\.mdx\/[^)]+)\)/g)];
|
||||
assert(markdownLinks.length > 0, "LLM discovery must link to exported Markdown");
|
||||
await Promise.all(
|
||||
markdownLinks.map(async ([, href]) => {
|
||||
const url = new URL(href, "https://homarr.dev");
|
||||
await access(path.join(outputDirectory, decodeURIComponent(url.pathname)));
|
||||
}),
|
||||
);
|
||||
console.log(`LLM discovery verified: ${markdownLinks.length} Markdown exports.`);
|
||||
console.log(`Search verified: page titles, headings, body content, and ${verifiedUrls.size} destinations.`);
|
||||
Reference in new issue
Block a user