interface Options> { requestAsync: (input: TInput, signal: AbortSignal) => Promise; getCacheKey?: (input: TInput) => string; cacheTtlMs?: number; fallbackToStaleOnError?: boolean; staleIfErrorTtlMs?: number; requestTimeoutMs?: number; getSharedCacheAsync?: (input: TInput, cacheIdentity: string) => Promise>; } export interface CacheEntry { data: TData; timestamp: Date; expiresAt: number; staleUntil: number; } export interface SharedCacheAdapter { generation: string; isShared: boolean; getAsync: () => Promise | null | undefined>; setAsync: (entry: CacheEntry, refreshLockToken: string) => Promise; acquireRefreshLockAsync: () => Promise; renewRefreshLockAsync: (token: string) => Promise; releaseRefreshLockAsync: (token: string) => Promise; } const MAX_CACHE_SIZE = 1000; const MAX_INFLIGHT_REQUESTS = 100; const MAX_UPSTREAM_SETTLEMENT_GRACE_MS = 60_000; const DEFAULT_TTL_MS = 10_000; const DEFAULT_STALE_IF_ERROR_TTL_MS = 5 * 60_000; const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; const SHARED_REFRESH_POLL_MS = 100; const SHARED_REFRESH_LOCK_RETRY_MS = 1_000; const SHARED_REFRESH_STALE_WAIT_MS = 15_000; const SHARED_REFRESH_MAX_WAIT_MS = 30_000; // Must remain comfortably below REFRESH_LOCK_TTL_SECONDS in shared-cache.ts. const SHARED_REFRESH_LOCK_RENEW_MS = 5_000; class RequestHandlerTimeoutError extends Error { constructor() { super("Request handler deadline exceeded"); this.name = RequestHandlerTimeoutError.name; } } class RequestHandlerOverloadedError extends Error { constructor() { super("Request handler concurrency limit exceeded"); this.name = RequestHandlerOverloadedError.name; } } const throwIfDeadlineExceeded = (deadlineAt: number) => { if (Date.now() >= deadlineAt) throw new RequestHandlerTimeoutError(); }; const delayAsync = async (durationMs: number, deadlineAt: number) => { throwIfDeadlineExceeded(deadlineAt); const remainingMs = deadlineAt - Date.now(); await new Promise((resolve) => { const timer = setTimeout(resolve, Math.min(durationMs, remainingMs)); timer.unref?.(); }); throwIfDeadlineExceeded(deadlineAt); }; const waitForSettlementGraceAsync = async (settledAsync: Promise, durationMs: number) => { await new Promise((resolve) => { const timer = setTimeout(resolve, durationMs); timer.unref?.(); void settledAsync.then(() => { clearTimeout(timer); resolve(); }); }); }; const startRefreshLockRenewal = (sharedCache: SharedCacheAdapter, token: string) => { let isOwned = true; let isStopped = false; let timer: ReturnType | undefined; const schedule = () => { timer = setTimeout(() => { void sharedCache.renewRefreshLockAsync(token).then( (renewed) => { if (renewed !== true) { isOwned = false; return; } if (!isStopped) schedule(); }, () => { isOwned = false; }, ); }, SHARED_REFRESH_LOCK_RENEW_MS); timer.unref?.(); }; schedule(); return { isOwned: () => isOwned, stop: () => { isStopped = true; if (timer) clearTimeout(timer); }, }; }; const evictExpired = (cache: Map>) => { const now = Date.now(); for (const [key, entry] of cache) { if (now >= entry.staleUntil) cache.delete(key); } }; export const createRequestHandler = >( options: Options, ) => { const requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs <= 0) { throw new Error("Request timeout must be a positive finite duration"); } const cache = new Map>(); const inflight = new Map>>(); const sharedCacheResolutions = new Map>>(); const upstreamSettlements = new Map>(); let expiryTimer: ReturnType | undefined; let expiryTimerAt: number | undefined; let generation = 0; const storeInMemory = (key: string, entry: CacheEntry, requestGeneration: number) => { if (generation !== requestGeneration || Date.now() >= entry.staleUntil) return; evictExpired(cache); cache.delete(key); if (cache.size >= MAX_CACHE_SIZE) { const oldest = cache.keys().next().value; if (oldest !== undefined) cache.delete(oldest); } cache.set(key, entry); scheduleExpiry(entry.staleUntil); }; const getFromMemory = (key: string) => { const entry = cache.get(key); if (!entry) return undefined; cache.delete(key); cache.set(key, entry); return entry; }; const startRequestWithDeadline = (input: TInput, deadlineAt: number, upstreamKey: string) => { throwIfDeadlineExceeded(deadlineAt); if (upstreamSettlements.has(upstreamKey) || upstreamSettlements.size >= MAX_INFLIGHT_REQUESTS) { throw new RequestHandlerOverloadedError(); } const controller = new AbortController(); const timeoutError = new RequestHandlerTimeoutError(); const remainingMs = deadlineAt - Date.now(); let upstreamRequest: Promise; try { upstreamRequest = Promise.resolve(options.requestAsync(input, controller.signal)); } catch (error) { upstreamRequest = Promise.reject(error); } const settledAsync = upstreamRequest.then( () => undefined, () => undefined, ); upstreamSettlements.set(upstreamKey, settledAsync); void settledAsync.then(() => { if (upstreamSettlements.get(upstreamKey) === settledAsync) upstreamSettlements.delete(upstreamKey); }); const resultAsync = new Promise((resolve, reject) => { const timer = setTimeout(() => { controller.abort(timeoutError); reject(timeoutError); }, remainingMs); timer.unref?.(); void upstreamRequest.then(resolve, reject).finally(() => clearTimeout(timer)); }); return { resultAsync, settledAsync }; }; const resolveSharedCacheAsync = (input: TInput, cacheIdentity: string, requestGeneration: number) => { if (!options.getSharedCacheAsync) return undefined; const resolutionKey = `${requestGeneration}:${cacheIdentity}`; const existing = sharedCacheResolutions.get(resolutionKey); if (existing) return existing; if (sharedCacheResolutions.size >= MAX_INFLIGHT_REQUESTS) { throw new RequestHandlerOverloadedError(); } let promise: Promise>; promise = options.getSharedCacheAsync(input, cacheIdentity).finally(() => { if (sharedCacheResolutions.get(resolutionKey) === promise) sharedCacheResolutions.delete(resolutionKey); }); sharedCacheResolutions.set(resolutionKey, promise); return promise; }; const scheduleExpiry = (candidateExpiryAt?: number) => { if (candidateExpiryAt !== undefined && expiryTimerAt !== undefined && expiryTimerAt <= candidateExpiryAt) return; if (expiryTimer) clearTimeout(expiryTimer); expiryTimer = undefined; expiryTimerAt = undefined; if (cache.size === 0) return; const nextExpiryAt = candidateExpiryAt ?? Math.min(...[...cache.values()].map(({ staleUntil }) => staleUntil)); expiryTimerAt = nextExpiryAt; expiryTimer = setTimeout( () => { expiryTimer = undefined; expiryTimerAt = undefined; evictExpired(cache); scheduleExpiry(); }, Math.max(0, nextExpiryAt - Date.now()), ); expiryTimer.unref?.(); }; return { invalidateCache: () => { generation += 1; cache.clear(); if (expiryTimer) clearTimeout(expiryTimer); expiryTimer = undefined; expiryTimerAt = undefined; }, handler: (input: TInput) => ({ async getDataAsync(): Promise<{ data: TData; timestamp: Date }> { const deadlineAt = Date.now() + requestTimeoutMs; const ttl = options.cacheTtlMs ?? DEFAULT_TTL_MS; const baseKey = options.getCacheKey?.(input) ?? JSON.stringify(input); const requestGeneration = generation; let sharedCache: SharedCacheAdapter | undefined; const sharedCacheResolution = ttl > 0 ? resolveSharedCacheAsync(input, baseKey, requestGeneration) : undefined; if (sharedCacheResolution) { sharedCache = await sharedCacheResolution; throwIfDeadlineExceeded(deadlineAt); } const key = `${sharedCache?.generation ?? "local"}:${baseKey}`; const inflightKey = `${requestGeneration}:${key}`; const existing = inflight.get(inflightKey); if (existing) return existing; let cached = getFromMemory(key); const now = Date.now(); if (cached && now < cached.expiresAt) return cached; if (cached && now >= cached.staleUntil) { cache.delete(key); scheduleExpiry(); cached = undefined; } if (upstreamSettlements.has(inflightKey) || inflight.size >= MAX_INFLIGHT_REQUESTS) { if (options.fallbackToStaleOnError && cached) return cached; throw new RequestHandlerOverloadedError(); } let cleanupAfterUpstreamAsync: Promise | undefined; let promise: Promise>; promise = (async () => { let stale = cached; let lockToken: string | null | undefined; if (sharedCache?.isShared) { const sharedEntry = await sharedCache.getAsync(); if (sharedEntry && Date.now() < sharedEntry.expiresAt) { storeInMemory(key, sharedEntry, requestGeneration); return sharedEntry; } if (sharedEntry && Date.now() < sharedEntry.staleUntil) stale = sharedEntry; lockToken = await sharedCache.acquireRefreshLockAsync(); if (lockToken === null) { const returnStaleAt = Date.now() + SHARED_REFRESH_STALE_WAIT_MS; const waitDeadlineAt = Math.min(deadlineAt, Date.now() + SHARED_REFRESH_MAX_WAIT_MS); let retryLockAt = Date.now() + SHARED_REFRESH_LOCK_RETRY_MS; while (lockToken === null && sharedCache.isShared) { await delayAsync(SHARED_REFRESH_POLL_MS, waitDeadlineAt); const refreshedEntry = await sharedCache.getAsync(); if (refreshedEntry && Date.now() < refreshedEntry.expiresAt) { storeInMemory(key, refreshedEntry, requestGeneration); return refreshedEntry; } if (refreshedEntry && Date.now() < refreshedEntry.staleUntil) stale = refreshedEntry; if (Date.now() >= returnStaleAt && options.fallbackToStaleOnError && stale) { storeInMemory(key, stale, requestGeneration); return stale; } if (Date.now() >= retryLockAt) { lockToken = await sharedCache.acquireRefreshLockAsync(); retryLockAt = Date.now() + SHARED_REFRESH_LOCK_RETRY_MS; } } } } const refreshLockRenewal = lockToken && sharedCache ? startRefreshLockRenewal(sharedCache, lockToken) : undefined; let upstreamSettledAsync: Promise | undefined; try { const request = startRequestWithDeadline(input, deadlineAt, inflightKey); upstreamSettledAsync = request.settledAsync; const data = await request.resultAsync; const completedAt = Date.now(); const staleIfErrorTtlMs = options.fallbackToStaleOnError ? Math.max(0, options.staleIfErrorTtlMs ?? DEFAULT_STALE_IF_ERROR_TTL_MS) : 0; const entry: CacheEntry = { data, timestamp: new Date(completedAt), expiresAt: completedAt + ttl, staleUntil: completedAt + ttl + staleIfErrorTtlMs, }; if (ttl > 0) { let canStoreInMemory = true; if (sharedCache?.isShared) { canStoreInMemory = false; if (lockToken && refreshLockRenewal?.isOwned() !== false) { const stored = await sharedCache.setAsync(entry, lockToken); if (stored !== false || !sharedCache.isShared) canStoreInMemory = true; } } if (canStoreInMemory) storeInMemory(key, entry, requestGeneration); } return entry; } catch (error) { if ( options.fallbackToStaleOnError && stale && Date.now() < stale.staleUntil && generation === requestGeneration ) { storeInMemory(key, stale, requestGeneration); return stale; } throw error; } finally { const cleanupAsync = async () => { refreshLockRenewal?.stop(); if (lockToken) await sharedCache?.releaseRefreshLockAsync(lockToken); }; if (upstreamSettledAsync) { const settlementGraceMs = Math.min(requestTimeoutMs, MAX_UPSTREAM_SETTLEMENT_GRACE_MS); cleanupAfterUpstreamAsync = waitForSettlementGraceAsync(upstreamSettledAsync, settlementGraceMs).then( cleanupAsync, ); } else { await cleanupAsync(); } } })(); inflight.set(inflightKey, promise); const removeInflight = () => { const cleanupAsync = cleanupAfterUpstreamAsync ?? Promise.resolve(); void cleanupAsync.then( () => { if (inflight.get(inflightKey) === promise) inflight.delete(inflightKey); }, () => { if (inflight.get(inflightKey) === promise) inflight.delete(inflightKey); }, ); }; void promise.then(removeInflight, removeInflight); return promise; }, }), }; };