feat(custom-widgets): complete Custom JSX v2 (#6390)

packages/custom-widgets/src/server/index.ts

@@ -5,6 +5,7 @@ export {
   assertSafeStaticHeaders,
   classifyAddress,
   executeCustomWidgetRequest,
+  invalidateCustomWidgetResponseCache,
   MAX_REQUEST_BODY_BYTES,
   MAX_RESPONSE_BODY_BYTES,
   MAX_RESPONSE_JSON_DEPTH,
@@ -24,5 +25,11 @@ export type {
   PreviewSessionServiceOptions,
   PreviewSessionStore,
 } from "./preview-sessions";
-export { hashRuntimeParams, renderRequestBody, renderRequestTarget, validateRuntimeParams } from "./request-manifest";
+export {
+  hashRuntimeParams,
+  renderBoundValue,
+  renderRequestBody,
+  renderRequestTarget,
+  resolveCustomWidgetRequestValues,
+} from "./request-manifest";
 export type { CustomJsxRuntimeParams } from "./request-manifest";

packages/custom-widgets/src/server/preview-sessions.ts

@@ -1,27 +1,35 @@
 import { z } from "zod/v4";
 
 import {
-  customJsxNetworkScopes,
-  customJsxRequestSchema,
-  customWidgetAuthTypes,
+  customWidgetOptionsSchema,
+  customWidgetRequestsSchema,
   customWidgetSecretKinds,
+  customWidgetSourcesSchema,
+  hasSameCustomWidgetSourceAuthentication,
 } from "../core";
-import type { CustomJsxNetworkScope, CustomJsxRequest, CustomWidgetAuthType } from "../core";
+import type { CustomJsxRequest, CustomWidgetOptions, CustomWidgetSource } from "../core";
 import { CustomWidgetDomainError } from "./errors";
 
-const SESSION_TTL_MS = 5 * 60_000;
+const SESSION_TTL_MS = 10 * 60_000;
 const MAX_JOURNAL_ENTRIES = 50;
 
+const encryptedSecretSchema = z.object({
+  sourceId: z.string(),
+  kind: z.enum(customWidgetSecretKinds),
+  value: z.string(),
+});
+
 const sessionSchema = z.object({
   id: z.string(),
   userId: z.string(),
   expiresAt: z.number(),
-  baseUrl: z.string(),
-  authType: z.enum(customWidgetAuthTypes),
-  headerName: z.string().nullable(),
-  secrets: z.array(z.object({ kind: z.enum(customWidgetSecretKinds), value: z.string() })),
-  networkScope: z.enum(customJsxNetworkScopes),
-  requests: z.array(customJsxRequestSchema),
+  sources: customWidgetSourcesSchema,
+  secrets: z.array(encryptedSecretSchema),
+  requests: customWidgetRequestsSchema,
+  name: z.string(),
+  template: z.string(),
+  optionDefinitions: customWidgetOptionsSchema,
+  options: z.record(z.string(), z.unknown()),
   definitionId: z.string().optional(),
   liveActions: z.boolean(),
 });
@@ -32,7 +40,7 @@ const journalEntrySchema = z.object({
   requestId: z.string(),
   kind: z.enum(["query", "action"]),
   method: z.enum(["GET", "POST", "PUT", "PATCH", "DELETE"]),
-  pathTemplate: z.string(),
+  path: z.string(),
   status: z.number().nullable(),
   durationMs: z.number().nonnegative(),
   simulated: z.boolean(),
@@ -42,12 +50,13 @@ export type CustomWidgetPreviewJournalEntry = z.infer<typeof journalEntrySchema>
 
 export interface CreatePreviewSessionInput {
   userId: string;
-  baseUrl: string;
-  authType: CustomWidgetAuthType;
-  headerName?: string;
-  secrets: Array<{ kind: (typeof customWidgetSecretKinds)[number]; value: string }>;
-  networkScope: CustomJsxNetworkScope;
-  requests: CustomJsxRequest[];
+  sources: Record<string, CustomWidgetSource>;
+  secrets: Array<{ sourceId: string; kind: (typeof customWidgetSecretKinds)[number]; value: string }>;
+  requests: Record<string, CustomJsxRequest>;
+  name: string;
+  template: string;
+  optionDefinitions: CustomWidgetOptions;
+  options: Record<string, unknown>;
   definitionId?: string;
 }
 
@@ -77,16 +86,24 @@ export class CustomWidgetPreviewSessionService {
   }
 
   public async create(input: CreatePreviewSessionInput) {
+    const sourceIds = new Set(Object.keys(input.sources));
+    if (input.secrets.some((secret) => !sourceIds.has(secret.sourceId))) {
+      throw new CustomWidgetDomainError({
+        code: "BAD_REQUEST",
+        message: "A preview secret references an unknown source",
+      });
+    }
     const session: CustomWidgetPreviewSession = {
       id: this.options.createId(),
       userId: input.userId,
       expiresAt: this.now() + SESSION_TTL_MS,
-      baseUrl: input.baseUrl,
-      authType: input.authType,
-      headerName: input.headerName ?? null,
-      secrets: input.secrets.map((secret) => ({ kind: secret.kind, value: this.options.encrypt(secret.value) })),
-      networkScope: input.networkScope,
+      sources: input.sources,
+      secrets: input.secrets.map((secret) => ({ ...secret, value: this.options.encrypt(secret.value) })),
       requests: input.requests,
+      name: input.name,
+      template: input.template,
+      optionDefinitions: input.optionDefinitions,
+      options: input.options,
       definitionId: input.definitionId,
       liveActions: false,
     };
@@ -132,8 +149,69 @@ export class CustomWidgetPreviewSessionService {
     });
   }
 
-  public getSecrets(session: CustomWidgetPreviewSession) {
-    return session.secrets.map((secret) => ({ kind: secret.kind, value: this.options.decrypt(secret.value) }));
+  public getSecrets(session: CustomWidgetPreviewSession, sourceId: string) {
+    return session.secrets
+      .filter((secret) => secret.sourceId === sourceId)
+      .map((secret) => ({ kind: secret.kind, value: this.options.decrypt(secret.value) }));
+  }
+
+  public async setSecrets(
+    id: string,
+    userId: string,
+    secrets: Array<{ sourceId: string; kind: (typeof customWidgetSecretKinds)[number]; value: string }>,
+  ) {
+    const session = await this.get(id, userId);
+    const sourceIds = new Set(Object.keys(session.sources));
+    if (secrets.some((secret) => !sourceIds.has(secret.sourceId))) {
+      throw new CustomWidgetDomainError({
+        code: "BAD_REQUEST",
+        message: "A preview secret references an unknown source",
+      });
+    }
+    const replacements = new Set(secrets.map((secret) => `${secret.sourceId}:${secret.kind}`));
+    const next = {
+      ...session,
+      secrets: [
+        ...session.secrets.filter((secret) => !replacements.has(`${secret.sourceId}:${secret.kind}`)),
+        ...secrets.map((secret) => ({ ...secret, value: this.options.encrypt(secret.value) })),
+      ],
+    };
+    await this.save(next);
+    return { id, expiresAt: next.expiresAt };
+  }
+
+  public async configureSource(
+    id: string,
+    userId: string,
+    sourceId: string,
+    source: CustomWidgetSource,
+    secrets: Array<{ sourceId: string; kind: (typeof customWidgetSecretKinds)[number]; value: string }>,
+  ) {
+    const session = await this.get(id, userId);
+    const currentSource = session.sources[sourceId];
+    if (!currentSource) {
+      throw new CustomWidgetDomainError({ code: "NOT_FOUND", message: "Preview source was not found" });
+    }
+    if (!hasSameCustomWidgetSourceAuthentication(currentSource, source)) {
+      throw new CustomWidgetDomainError({
+        code: "BAD_REQUEST",
+        message: "Preview source authentication changed; create a new configuration request",
+      });
+    }
+    const replacements = new Set(secrets.map((secret) => `${secret.sourceId}:${secret.kind}`));
+    const next = {
+      ...session,
+      sources: {
+        ...session.sources,
+        [sourceId]: { ...currentSource, baseUrl: source.baseUrl, networkScope: source.networkScope },
+      },
+      secrets: [
+        ...session.secrets.filter((secret) => !replacements.has(`${secret.sourceId}:${secret.kind}`)),
+        ...secrets.map((secret) => ({ ...secret, value: this.options.encrypt(secret.value) })),
+      ],
+    };
+    await this.save(next);
+    return { id, expiresAt: next.expiresAt };
   }
 
   private async save(session: CustomWidgetPreviewSession): Promise<void> {

packages/custom-widgets/src/server/request-executor.ts

@@ -1,6 +1,6 @@
 import { Buffer } from "node:buffer";
-import { fetch, Headers } from "undici";
-import type { Response } from "undici";
+import { STATUS_CODES } from "node:http";
+import { Headers, Response } from "undici";
 
 import type { CustomJsxNetworkScope, CustomWidgetMethod } from "../core";
 import { applyAuth } from "./auth";
@@ -61,36 +61,39 @@ export interface CustomWidgetHttpResponse {
 }
 const cache = new Map<string, { expiresAt: number; response: CustomWidgetHttpResponse }>();
 const inFlight = new Map<string, Promise<CustomWidgetHttpResponse>>();
+let cacheEpoch = 0;
 
 async function performRequest(input: CustomWidgetHttpRequest): Promise<CustomWidgetHttpResponse> {
   assertRequest(input);
   const baseUrl = validateCustomWidgetUrl(input.baseUrl);
   let currentUrl = resolveSameOriginTarget(input.baseUrl, input.targetUrl);
+  let currentMethod = input.method;
+  let currentBody = input.body;
   const maxRedirects = input.kind === "query" ? MAX_QUERY_REDIRECTS : 0;
   for (let redirects = 0; ; redirects += 1) {
     const dispatcher = createPinnedAgent(
       await resolveAndValidateHost(currentUrl.hostname, input.networkScope),
       REQUEST_TIMEOUT_MS,
     );
-    const headers = buildHeaders(input, currentUrl);
+    const headers = buildHeaders(input, currentUrl, currentBody);
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
-    let response: Response;
+    let responseData: Awaited<ReturnType<typeof dispatcher.request>>;
     try {
-      response = await fetch(currentUrl, {
-        method: input.method,
+      responseData = await dispatcher.request({
+        origin: currentUrl.origin,
+        path: `${currentUrl.pathname}${currentUrl.search}`,
+        method: currentMethod,
         headers,
-        body: input.method === "GET" ? undefined : input.body,
-        redirect: "manual",
+        body: currentBody,
         signal: controller.signal,
-        dispatcher,
       });
     } catch (error) {
       await dispatcher.close();
       if (error instanceof CustomWidgetDomainError) throw error;
       input.logError?.({
         origin: currentUrl.origin,
-        method: input.method,
+        method: currentMethod,
         errorName: error instanceof Error ? error.name : "UnknownError",
       });
       throw new CustomWidgetDomainError({
@@ -101,8 +104,14 @@ async function performRequest(input: CustomWidgetHttpRequest): Promise<CustomWid
     } finally {
       clearTimeout(timeout);
     }
-    if (![301, 302, 303, 307, 308].includes(response.status)) {
+    if (![301, 302, 303, 307, 308].includes(responseData.statusCode)) {
       try {
+        const body = await responseData.body.arrayBuffer();
+        const response = new Response(body.byteLength > 0 ? body : null, {
+          status: responseData.statusCode,
+          statusText: STATUS_CODES[responseData.statusCode] ?? "",
+          headers: normalizeResponseHeaders(responseData.headers),
+        });
         return {
           ok: response.ok,
           status: response.status,
@@ -113,34 +122,47 @@ async function performRequest(input: CustomWidgetHttpRequest): Promise<CustomWid
         await dispatcher.close();
       }
     }
-    await response.body?.cancel();
-    await dispatcher.close();
+    try {
+      await responseData.body.dump();
+    } finally {
+      await dispatcher.close();
+    }
     if (redirects >= maxRedirects)
       throw new CustomWidgetDomainError({ code: "BAD_GATEWAY", message: "Upstream redirect limit exceeded" });
-    const location = response.headers.get("location");
+    const location = normalizeResponseHeaders(responseData.headers).get("location");
     if (!location)
       throw new CustomWidgetDomainError({ code: "BAD_GATEWAY", message: "Upstream redirect is missing a location" });
     const redirected = validateCustomWidgetUrl(new URL(location, currentUrl));
     if (redirected.origin !== baseUrl.origin)
       throw new CustomWidgetDomainError({ code: "FORBIDDEN", message: "Cross-origin redirects are not allowed" });
+    if (responseData.statusCode === 303 || ([301, 302].includes(responseData.statusCode) && currentMethod === "POST")) {
+      currentMethod = "GET";
+      currentBody = undefined;
+    }
     currentUrl = redirected;
   }
 }
 
+function normalizeResponseHeaders(values: Record<string, string | string[] | undefined>) {
+  const headers = new Headers();
+  for (const [name, value] of Object.entries(values)) {
+    if (Array.isArray(value)) value.forEach((entry) => headers.append(name, entry));
+    else if (value !== undefined) headers.set(name, value);
+  }
+  return headers;
+}
+
 function assertRequest(input: CustomWidgetHttpRequest): void {
-  if (input.kind === "query" && input.method !== "GET")
-    throw new CustomWidgetDomainError({ code: "BAD_REQUEST", message: "Queries must use GET" });
-  if (input.kind === "action" && input.method === "GET")
-    throw new CustomWidgetDomainError({ code: "BAD_REQUEST", message: "Actions cannot use GET" });
   if (input.body !== undefined && Buffer.byteLength(input.body, "utf8") > MAX_REQUEST_BODY_BYTES)
     throw new CustomWidgetDomainError({ code: "PAYLOAD_TOO_LARGE", message: "Request body exceeds the 10 KiB limit" });
   assertSafeStaticHeaders(input.staticHeaders);
 }
 
-function buildHeaders(input: CustomWidgetHttpRequest, url: URL): Headers {
+function buildHeaders(input: CustomWidgetHttpRequest, url: URL, body: string | undefined): Headers {
   const headers = new Headers({ Accept: "application/json" });
   for (const [name, value] of Object.entries(input.staticHeaders ?? {})) headers.set(name, value);
-  if (input.body !== undefined) headers.set("Content-Type", "application/json");
+  if (body !== undefined) headers.set("Content-Type", "application/json");
+  else headers.delete("Content-Type");
   if (input.auth) {
     if (input.auth.type === "apiKeyHeader") assertSafeStaticHeaders({ [input.auth.headerName ?? "X-API-Key"]: "" });
     applyAuth(headers, url, input.auth.type, input.auth.secrets, input.auth.headerName);
@@ -149,26 +171,37 @@ function buildHeaders(input: CustomWidgetHttpRequest, url: URL): Headers {
 }
 
 export async function executeCustomWidgetRequest(input: CustomWidgetHttpRequest): Promise<CustomWidgetHttpResponse> {
-  const key = input.method === "GET" ? input.cacheKey : undefined;
+  const key = input.kind === "query" ? input.cacheKey : undefined;
   if (!key) return performRequest(input);
   const cached = cache.get(key);
   if (cached && cached.expiresAt > Date.now()) return cached.response;
   if (cached) cache.delete(key);
   const pending = inFlight.get(key);
   if (pending) return pending;
-  const request = performRequest(input)
+  const requestEpoch = cacheEpoch;
+  let request: Promise<CustomWidgetHttpResponse>;
+  request = performRequest(input)
     .then((response) => {
-      if (response.ok && (input.cacheTtlSeconds ?? 0) > 0) {
+      if (response.ok && (input.cacheTtlSeconds ?? 0) > 0 && requestEpoch === cacheEpoch) {
         pruneCache();
         cache.set(key, { expiresAt: Date.now() + (input.cacheTtlSeconds ?? 0) * 1000, response });
       }
       return response;
     })
-    .finally(() => inFlight.delete(key));
+    .finally(() => {
+      if (inFlight.get(key) === request) inFlight.delete(key);
+    });
   inFlight.set(key, request);
   return request;
 }
 
+export function invalidateCustomWidgetResponseCache(prefixes: readonly string[]): void {
+  if (prefixes.length === 0) return;
+  cacheEpoch += 1;
+  for (const key of cache.keys()) if (prefixes.some((prefix) => key.startsWith(prefix))) cache.delete(key);
+  for (const key of inFlight.keys()) if (prefixes.some((prefix) => key.startsWith(prefix))) inFlight.delete(key);
+}
+
 function pruneCache(): void {
   for (const [key, entry] of cache) if (entry.expiresAt <= Date.now()) cache.delete(key);
   while (cache.size >= MAX_RESPONSE_CACHE_ENTRIES) {

packages/custom-widgets/src/server/request-manifest.ts

@@ -1,79 +1,107 @@
 import { createHash } from "node:crypto";
 
-import { CustomWidgetDomainError } from "./errors";
-
 import type { CustomJsxRequest } from "../core";
-
+import { collectCustomWidgetRequestReferences } from "../core";
+import { CustomWidgetDomainError } from "./errors";
 import { resolveSameOriginTarget } from "./request-executor";
 
 export type CustomJsxRuntimeParams = Record<string, string | number | boolean>;
+type CustomJsxResolvedValues = Record<string, unknown>;
 
-export const validateRuntimeParams = (request: CustomJsxRequest, params: CustomJsxRuntimeParams) => {
-  const declaredNames = Object.keys(request.parameters).toSorted();
-  const suppliedNames = Object.keys(params).toSorted();
+export function resolveCustomWidgetRequestValues(
+  request: CustomJsxRequest,
+  options: Record<string, unknown>,
+  suppliedParams: CustomJsxRuntimeParams = {},
+): CustomJsxResolvedValues {
+  const references = collectCustomWidgetRequestReferences(request);
+  const suppliedNames = Object.keys(suppliedParams).toSorted();
+  const expectedNames = [...references.params].toSorted();
   if (
-    declaredNames.length !== suppliedNames.length ||
-    declaredNames.some((name, index) => name !== suppliedNames[index])
+    suppliedNames.length !== expectedNames.length ||
+    suppliedNames.some((name, index) => name !== expectedNames[index])
   ) {
-    throw new CustomWidgetDomainError({ code: "BAD_REQUEST", message: "Request parameters do not match the manifest" });
+    throw new CustomWidgetDomainError({
+      code: "BAD_REQUEST",
+      message: "Request parameters do not match the request references",
+    });
   }
-  for (const [name, expectedType] of Object.entries(request.parameters)) {
-    if (typeof params[name] !== expectedType) {
+  const values: CustomJsxResolvedValues = { ...suppliedParams };
+  for (const name of references.options) {
+    const value = options[name];
+    if (value === undefined) {
       throw new CustomWidgetDomainError({
         code: "BAD_REQUEST",
-        message: `Request parameter '${name}' must be ${expectedType}`,
+        message: `Widget option '${name}' is missing`,
       });
     }
+    values[`option:${name}`] = value;
   }
-};
+  return values;
+}
 
-export const renderRequestBody = (
-  template: CustomJsxRequest["bodyTemplate"],
-  params: CustomJsxRuntimeParams,
-): string | undefined => {
-  if (template === undefined) return undefined;
+export const renderBoundValue = (template: unknown, values: CustomJsxResolvedValues): unknown => {
   const substitute = (value: unknown): unknown => {
     if (Array.isArray(value)) return value.map(substitute);
     if (value !== null && typeof value === "object") {
       const entries = Object.entries(value);
-      if (entries.length === 1 && entries[0]?.[0] === "$param" && typeof entries[0][1] === "string") {
-        const parameter = params[entries[0][1]];
-        if (parameter === undefined) {
+      if (
+        entries.length === 1 &&
+        ["$param", "$option"].includes(entries[0]?.[0] ?? "") &&
+        typeof entries[0]?.[1] === "string"
+      ) {
+        const prefix = entries[0][0] === "$option" ? "option:" : "";
+        const resolved = values[`${prefix}${entries[0][1]}`];
+        if (resolved === undefined)
           throw new CustomWidgetDomainError({
             code: "BAD_REQUEST",
-            message: `Unknown body parameter '${entries[0][1]}'`,
+            message: `Unknown request value '${entries[0][1]}'`,
           });
-        }
-        return parameter;
+        return resolved;
       }
       return Object.fromEntries(entries.map(([key, child]) => [key, substitute(child)]));
     }
     return value;
   };
-  return JSON.stringify(substitute(template));
+  return substitute(template);
 };
 
-export const renderRequestTarget = (baseUrl: string, request: CustomJsxRequest, params: CustomJsxRuntimeParams) => {
-  validateRuntimeParams(request, params);
-  const path = request.pathTemplate.replaceAll(/\{([a-z][a-z0-9_-]*)\}/g, (_placeholder, name: string) => {
-    const value = params[name];
-    if (value === undefined) {
-      throw new CustomWidgetDomainError({ code: "BAD_REQUEST", message: `Unknown path parameter '${name}'` });
-    }
-    return encodeURIComponent(String(value));
-  });
-  if (/[{}]/.test(path)) {
-    throw new CustomWidgetDomainError({
-      code: "BAD_REQUEST",
-      message: "Path template contains an invalid placeholder",
-    });
-  }
-  return resolveSameOriginTarget(baseUrl, new URL(path, baseUrl));
-};
+export const renderRequestBody = (request: CustomJsxRequest, values: CustomJsxResolvedValues): string | undefined =>
+  request.body === undefined ? undefined : JSON.stringify(renderBoundValue(request.body, values));
 
-export const hashRuntimeParams = (params: CustomJsxRuntimeParams) => {
-  const stableParams = Object.fromEntries(
-    Object.entries(params).toSorted(([left], [right]) => left.localeCompare(right)),
+export const renderRequestTarget = (baseUrl: string, request: CustomJsxRequest, values: CustomJsxResolvedValues) => {
+  const path = request.path.replaceAll(
+    /\{(option|param):([A-Za-z][A-Za-z0-9_-]*)\}/gu,
+    (_match, kind: string, name: string) => {
+      const value = values[kind === "option" ? `option:${name}` : name];
+      if (value === undefined)
+        throw new CustomWidgetDomainError({ code: "BAD_REQUEST", message: `Unknown path value '${name}'` });
+      if (!["string", "number", "boolean"].includes(typeof value))
+        throw new CustomWidgetDomainError({
+          code: "BAD_REQUEST",
+          message: `Path value '${name}' must resolve to text, number, or boolean`,
+        });
+      return encodeURIComponent(String(value));
+    },
   );
-  return createHash("sha256").update(JSON.stringify(stableParams)).digest("hex");
+  if (/[{}]/u.test(path))
+    throw new CustomWidgetDomainError({ code: "BAD_REQUEST", message: "Request path contains an invalid placeholder" });
+  const base = new URL(baseUrl);
+  base.pathname = `${base.pathname.replace(/\/$/u, "")}${path}` || "/";
+  const target = resolveSameOriginTarget(baseUrl, base);
+  for (const [name, template] of Object.entries(request.query ?? {})) {
+    const value = renderBoundValue(template, values);
+    if (!["string", "number", "boolean"].includes(typeof value)) {
+      throw new CustomWidgetDomainError({
+        code: "BAD_REQUEST",
+        message: `Query value '${name}' must resolve to text, number, or boolean`,
+      });
+    }
+    target.searchParams.set(name, String(value));
+  }
+  return target;
 };
+
+export function hashRuntimeParams(params: CustomJsxResolvedValues) {
+  const stable = Object.fromEntries(Object.entries(params).toSorted(([left], [right]) => left.localeCompare(right)));
+  return createHash("sha256").update(JSON.stringify(stable)).digest("hex");
+}