Compare commits

...
Author SHA1 Message Date
ajnart f7dae52b7e style: auto-fix lint and format 2026-06-01 15:43:07 +00:00
Thomas Camlong 0f3eabfb11 perf(integrations): speed up mediaRequestList Overseerr fetches
Reuse a shared TLS agent with timeouts, parallelize list calls, dedupe
detail fetches, and cap concurrency to reduce cron job duration.
2026-06-01 17:41:53 +02:00
8 changed files with 210 additions and 178 deletions

No files matched your search

@@ -149,7 +149,8 @@ export const NewIntegrationForm = ({ searchParams }: NewIntegrationFormProps) =>
};
const integrationCategories = integrationDefs[searchParams.kind].category.flat();
const supportsSearchEngine = integrationCategories.includes("search") && !integrationCategories.includes("mediaSearch");
const supportsSearchEngine =
integrationCategories.includes("search") && !integrationCategories.includes("mediaSearch");
return (
<form onSubmit={form.onSubmit((value) => void handleSubmitAsync(value))}>
@@ -82,7 +82,7 @@ export const integrationRouter = createTRPCRouter({
},
};
})
.sort(
.toSorted(
(integrationA, integrationB) =>
integrationKinds.indexOf(integrationA.kind) - integrationKinds.indexOf(integrationB.kind),
);
@@ -130,7 +130,7 @@ export const integrationRouter = createTRPCRouter({
},
};
})
.sort(
.toSorted(
(integrationA, integrationB) =>
integrationKinds.indexOf(integrationA.kind) - integrationKinds.indexOf(integrationB.kind),
);
@@ -181,7 +181,7 @@ export const integrationRouter = createTRPCRouter({
},
};
})
.sort(
.toSorted(
(integrationA, integrationB) =>
integrationKinds.indexOf(integrationA.kind) - integrationKinds.indexOf(integrationB.kind),
);
@@ -484,7 +484,7 @@ export const integrationRouter = createTRPCRouter({
});
return {
inherited: dbGroupPermissions.sort((permissionA, permissionB) => {
inherited: dbGroupPermissions.toSorted((permissionA, permissionB) => {
return permissionA.group.name.localeCompare(permissionB.group.name);
}),
users: userPermissions
@@ -492,7 +492,7 @@ export const integrationRouter = createTRPCRouter({
user,
permission,
}))
.sort((permissionA, permissionB) => {
.toSorted((permissionA, permissionB) => {
return (permissionA.user.name ?? "").localeCompare(permissionB.user.name ?? "");
}),
groups: dbGroupIntegrationPermission
@@ -503,7 +503,7 @@ export const integrationRouter = createTRPCRouter({
},
permission,
}))
.sort((permissionA, permissionB) => {
.toSorted((permissionA, permissionB) => {
return permissionA.group.name.localeCompare(permissionB.group.name);
}),
};
@@ -14,7 +14,9 @@ export const mediaRequestStatsJob = createCronJob("mediaRequestStats", EVERY_MIN
}),
);
export const mediaRequestListJob = createCronJob("mediaRequestList", EVERY_MINUTE).withCallback(
export const mediaRequestListJob = createCronJob("mediaRequestList", EVERY_MINUTE, {
expectedMaximumDurationInMillis: 15_000,
}).withCallback(
createRequestIntegrationJobHandler(mediaRequestListRequestHandler.handler, {
widgetKinds: ["mediaRequests-requestList"],
getInput: {
@@ -1,6 +1,10 @@
import { z } from "zod/v4";
import type { RequestInit, Response } from "undici";
import { fetch } from "undici";
import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http";
import { splitToChunksWithNItems } from "@homarr/common";
import { createCertificateAgentAsync } from "@homarr/core/infrastructure/http";
import { withTimeoutAsync } from "@homarr/core/infrastructure/http/timeout";
import { createLogger } from "@homarr/core/infrastructure/logs";
import { ErrorWithMetadata } from "@homarr/core/infrastructure/logs/error";
@@ -24,6 +28,14 @@ import {
const logger = createLogger({ module: "overseerrIntegration" });
const REQUEST_TIMEOUT_MS = 10_000;
const DETAIL_FETCH_BATCH_SIZE = 5;
interface OverseerrFetchClient {
fetchAsync: (url: URL, init?: RequestInit) => Promise<Response>;
fetchUncheckedAsync: (url: URL, init?: RequestInit) => Promise<Response>;
}
interface OverseerrSearchResult {
id: number;
name: string;
@@ -40,12 +52,43 @@ export class OverseerrIntegration
extends Integration
implements IMediaRequestIntegration, ISearchableIntegration<OverseerrSearchResult>
{
public async searchAsync(query: string) {
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/search", { query }), {
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
private async createFetchClientAsync(): Promise<OverseerrFetchClient> {
const agent = await createCertificateAgentAsync(undefined);
const apiKey = this.getSecretValue("apiKey");
const requestAsync = async (url: URL, init?: RequestInit) => {
return await withTimeoutAsync(
(signal) =>
fetch(url, {
...init,
signal,
dispatcher: agent,
headers: {
"X-Api-Key": apiKey,
...init?.headers,
},
}),
REQUEST_TIMEOUT_MS,
);
};
return {
fetchAsync: async (url, init) => {
const response = await requestAsync(url, init);
if (!response.ok) {
throw new Error(`Overseerr request failed: ${response.status} ${response.statusText} (${url.toString()})`);
}
return response;
},
});
fetchUncheckedAsync: requestAsync,
};
}
public async searchAsync(query: string) {
const client = await this.createFetchClientAsync();
const response = await client.fetchAsync(this.url("/api/v1/search", { query }));
const schemaData = await searchSchema.parseAsync(await response.json());
if (!schemaData.results) {
@@ -67,17 +110,12 @@ export class OverseerrIntegration
}
public async getSeriesInformationAsync(mediaType: "movie" | "tv", id: number) {
const client = await this.createFetchClientAsync();
const url = mediaType === "tv" ? this.url(`/api/v1/tv/${id}`) : this.url(`/api/v1/movie/${id}`);
const response = await fetchWithTrustedCertificatesAsync(url, {
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
},
});
const response = await client.fetchAsync(url);
const data = await mediaInformationSchema.parseAsync(await response.json());
const requestedSeasons = [
...new Set(
data.mediaInfo?.requests?.flatMap((req) => req.seasons.map((s) => s.seasonNumber)) ?? [],
),
...new Set(data.mediaInfo?.requests?.flatMap((req) => req.seasons.map((s) => s.seasonNumber)) ?? []),
];
const { mediaInfo: _strip, ...rest } = data;
return { ...rest, requestedSeasons };
@@ -90,8 +128,9 @@ export class OverseerrIntegration
* @param seasons A list of the seasons that should be requested.
*/
public async requestMediaAsync(mediaType: "movie" | "tv", id: number, seasons?: number[]): Promise<void> {
const client = await this.createFetchClientAsync();
const url = this.url("/api/v1/request");
const response = await fetchWithTrustedCertificatesAsync(url, {
const response = await client.fetchUncheckedAsync(url, {
method: "POST",
body: JSON.stringify({
mediaType,
@@ -99,7 +138,6 @@ export class OverseerrIntegration
seasons,
}),
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
"Content-Type": "application/json",
},
});
@@ -125,25 +163,16 @@ export class OverseerrIntegration
}
public async getRequestsAsync(): Promise<MediaRequest[]> {
const pendingRequests = await fetchWithTrustedCertificatesAsync(
this.url("/api/v1/request", { take: 20, filter: "pending" }),
{
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
},
},
);
const client = await this.createFetchClientAsync();
const allRequests = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/request", { take: 20 }), {
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
},
});
const [pendingResponse, allResponse] = await Promise.all([
client.fetchAsync(this.url("/api/v1/request", { take: 20, filter: "pending" })),
client.fetchAsync(this.url("/api/v1/request", { take: 20 })),
]);
const pendingResults = (await getRequestsSchema.parseAsync(await pendingRequests.json())).results;
const allResults = (await getRequestsSchema.parseAsync(await allRequests.json())).results;
const pendingResults = (await getRequestsSchema.parseAsync(await pendingResponse.json())).results;
const allResults = (await getRequestsSchema.parseAsync(await allResponse.json())).results;
//Concat the 2 lists while remove any duplicate pending from the all items list
let requests;
if (pendingResults.length > 0 && allResults.length > 0) {
@@ -152,37 +181,53 @@ export class OverseerrIntegration
);
} else if (pendingResults.length > 0) requests = pendingResults;
else if (allResults.length > 0) requests = allResults;
else return Promise.all([]);
else return [];
return await Promise.all(
requests.map(async (request): Promise<MediaRequest> => {
const information = await this.getItemInformationAsync(request.media.tmdbId, request.type);
const uniqueKeys = [...new Set(requests.map((request) => `${request.type}:${request.media.tmdbId}`))];
const informationByKey = new Map<string, MediaInformation>();
// See https://github.com/seerr-team/seerr/blob/af083a3cd5c3e3d5d7917fdf4fdd67fe3f39c46b/src/components/StatusBadge/index.tsx#L40
const inProgress = (request.media.downloadStatus ?? []).length >= 1;
for (const batch of splitToChunksWithNItems(uniqueKeys, DETAIL_FETCH_BATCH_SIZE)) {
const batchResults = await Promise.all(
batch.map(async (key) => {
const separatorIndex = key.indexOf(":");
const type = key.slice(0, separatorIndex) as MediaRequest["type"];
const tmdbId = Number(key.slice(separatorIndex + 1));
const information = await this.getItemInformationAsync(client, type, tmdbId);
return [key, information] as const;
}),
);
return {
id: request.id,
name: information.name,
status: this.mapRequestStatus(request.status),
availability: this.mapAvailability(request.media.status, inProgress),
backdropImageUrl: `https://image.tmdb.org/t/p/original/${information.backdropPath}`,
posterImagePath: `https://image.tmdb.org/t/p/w600_and_h900_bestv2/${information.posterPath}`,
href: this.externalUrl(`/${request.type}/${request.media.tmdbId}`).toString(),
type: request.type,
createdAt: request.createdAt,
airDate: new Date(information.airDate),
requestedBy: request.requestedBy
? ({
...request.requestedBy,
displayName: request.requestedBy.displayName,
link: this.externalUrl(`/users/${request.requestedBy.id}`).toString(),
avatar: this.constructAvatarUrl(request.requestedBy.avatar).toString(),
} satisfies Omit<RequestUser, "requestCount">)
: undefined,
};
}),
);
for (const [key, information] of batchResults) {
informationByKey.set(key, information);
}
}
return requests.map((request): MediaRequest => {
const information = informationByKey.get(`${request.type}:${request.media.tmdbId}`)!;
const inProgress = (request.media.downloadStatus ?? []).length >= 1;
return {
id: request.id,
name: information.name,
status: this.mapRequestStatus(request.status),
availability: this.mapAvailability(request.media.status, inProgress),
backdropImageUrl: `https://image.tmdb.org/t/p/original/${information.backdropPath}`,
posterImagePath: `https://image.tmdb.org/t/p/w600_and_h900_bestv2/${information.posterPath}`,
href: this.externalUrl(`/${request.type}/${request.media.tmdbId}`).toString(),
type: request.type,
createdAt: request.createdAt,
airDate: new Date(information.airDate),
requestedBy: request.requestedBy
? ({
...request.requestedBy,
displayName: request.requestedBy.displayName,
link: this.externalUrl(`/users/${request.requestedBy.id}`).toString(),
avatar: this.constructAvatarUrl(request.requestedBy.avatar).toString(),
} satisfies Omit<RequestUser, "requestCount">)
: undefined,
};
});
}
protected mapRequestStatus(status: UpstreamMediaRequestStatus): MediaRequestStatus {
@@ -222,20 +267,14 @@ export class OverseerrIntegration
}
public async getStatsAsync(): Promise<RequestStats> {
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/request/count"), {
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
},
});
const client = await this.createFetchClientAsync();
const response = await client.fetchAsync(this.url("/api/v1/request/count"));
return await getStatsSchema.parseAsync(await response.json());
}
public async getUsersAsync(): Promise<RequestUser[]> {
const response = await fetchWithTrustedCertificatesAsync(this.url("/api/v1/user", { take: 10, sort: "requests" }), {
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
},
});
const client = await this.createFetchClientAsync();
const response = await client.fetchAsync(this.url("/api/v1/user", { take: 10, sort: "requests" }));
const users = (await getUsersSchema.parseAsync(await response.json())).results;
return users.map((user): RequestUser => {
return {
@@ -248,75 +287,58 @@ export class OverseerrIntegration
public async approveRequestAsync(requestId: number): Promise<void> {
logger.info("Approving media request", { requestId, integration: this.integration.name });
await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/request/${requestId}/approve`), {
method: "POST",
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
},
}).then((response) => {
if (!response.ok) {
logger.error(
new ErrorWithMetadata("Failed to approve media request", {
requestId,
integration: this.integration.name,
reason: `${response.status} ${response.statusText}`,
url: response.url,
}),
);
}
const client = await this.createFetchClientAsync();
await client
.fetchUncheckedAsync(this.url(`/api/v1/request/${requestId}/approve`), {
method: "POST",
})
.then((response) => {
if (!response.ok) {
logger.error(
new ErrorWithMetadata("Failed to approve media request", {
requestId,
integration: this.integration.name,
reason: `${response.status} ${response.statusText}`,
url: response.url,
}),
);
}
logger.info("Successfully approved media request", { requestId, integration: this.integration.name });
});
logger.info("Successfully approved media request", { requestId, integration: this.integration.name });
});
}
public async declineRequestAsync(requestId: number): Promise<void> {
logger.info("Declining media request", { requestId, integration: this.integration.name });
const client = await this.createFetchClientAsync();
await client
.fetchUncheckedAsync(this.url(`/api/v1/request/${requestId}/decline`), {
method: "POST",
})
.then((response) => {
if (!response.ok) {
logger.error(
new ErrorWithMetadata("Failed to decline media request", {
requestId,
integration: this.integration.name,
reason: `${response.status} ${response.statusText}`,
url: response.url,
}),
);
}
await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/request/${requestId}/decline`), {
method: "POST",
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
},
}).then((response) => {
if (!response.ok) {
logger.error(
new ErrorWithMetadata("Failed to decline media request", {
requestId,
integration: this.integration.name,
reason: `${response.status} ${response.statusText}`,
url: response.url,
}),
);
}
logger.info("Successfully declined media request", { requestId, integration: this.integration.name });
});
logger.info("Successfully declined media request", { requestId, integration: this.integration.name });
});
}
private async getItemInformationAsync(id: number, type: MediaRequest["type"]): Promise<MediaInformation> {
const response = await fetchWithTrustedCertificatesAsync(this.url(`/api/v1/${type}/${id}`), {
headers: {
"X-Api-Key": this.getSecretValue("apiKey"),
},
});
if (type === "tv") {
const series = (await response.json()) as TvInformation;
return {
name: series.name,
backdropPath: series.backdropPath ?? series.posterPath,
posterPath: series.posterPath ?? series.backdropPath,
airDate: series.firstAirDate,
} satisfies MediaInformation;
}
const movie = (await response.json()) as MovieInformation;
return {
name: movie.title,
backdropPath: movie.backdropPath ?? movie.posterPath,
posterPath: movie.posterPath ?? movie.backdropPath,
airDate: movie.releaseDate,
} satisfies MediaInformation;
private async getItemInformationAsync(
client: OverseerrFetchClient,
type: MediaRequest["type"],
id: number,
): Promise<MediaInformation> {
const response = await client.fetchAsync(this.url(`/api/v1/${type}/${id}`));
const rawData = (await response.json()) as TvInformation | MovieInformation;
return mediaInformationFromResponse[type](rawData as never);
}
private constructAvatarUrl(avatar: string) {
@@ -351,13 +373,36 @@ interface MovieInformation {
releaseDate: string;
}
const mediaInfoRequestsSchema = z.object({
requests: z.array(z.object({
seasons: z.array(z.object({
seasonNumber: z.number(),
})),
})).optional(),
}).optional();
const mediaInformationFromResponse = {
tv: (series: TvInformation): MediaInformation => ({
name: series.name,
backdropPath: series.backdropPath ?? series.posterPath,
posterPath: series.posterPath ?? series.backdropPath,
airDate: series.firstAirDate,
}),
movie: (movie: MovieInformation): MediaInformation => ({
name: movie.title,
backdropPath: movie.backdropPath ?? movie.posterPath,
posterPath: movie.posterPath ?? movie.backdropPath,
airDate: movie.releaseDate,
}),
} satisfies Record<MediaRequest["type"], (data: TvInformation | MovieInformation) => MediaInformation>;
const mediaInfoRequestsSchema = z
.object({
requests: z
.array(
z.object({
seasons: z.array(
z.object({
seasonNumber: z.number(),
}),
),
}),
)
.optional(),
})
.optional();
const mediaInformationSchema = z.union([
z.object({
@@ -383,9 +428,11 @@ const mediaInformationSchema = z.union([
}),
]);
const searchMediaInfoSchema = z.object({
status: z.number(),
}).optional();
const searchMediaInfoSchema = z
.object({
status: z.number(),
})
.optional();
const searchSchema = z.object({
results: z
@@ -78,12 +78,8 @@ export const homeSearchEngineGroup = createGroup<GroupItem>({
return {
isLoading:
defaultSearchEngineQuery.isLoading ||
(resultQuery.isLoading && fromIntegrationEnabled) ||
status === "loading",
isError:
defaultSearchEngineQuery.isError ||
(resultQuery.isError && fromIntegrationEnabled),
defaultSearchEngineQuery.isLoading || (resultQuery.isLoading && fromIntegrationEnabled) || status === "loading",
isError: defaultSearchEngineQuery.isError || (resultQuery.isError && fromIntegrationEnabled),
data: createDefaultSearchEntries(defaultSearchEngine, results, session, query, t),
};
},
@@ -72,7 +72,11 @@ export const mediaRequestSearchGroup = createGroup<MediaRequestSearchOption>({
<Stack gap={2} style={{ flex: 1 }}>
<Group gap="xs" wrap="nowrap">
<Text>{result.name}</Text>
{badgeLabel && <Badge size="xs" color={badgeColor} variant="light">{badgeLabel}</Badge>}
{badgeLabel && (
<Badge size="xs" color={badgeColor} variant="light">
{badgeLabel}
</Badge>
)}
</Group>
<Text c="dimmed" size="sm" lineClamp={2}>
{result.text ?? getIntegrationName(result.integration.kind)}
@@ -58,7 +58,7 @@ export default function MediaServerWidget({
const newData = filteredData.concat(
data.requests.map((request) => ({ ...request, integrationId: data.integrationId })),
);
return newData.sort((dataA, dataB) => {
return newData.toSorted((dataA, dataB) => {
if (dataA.status === dataB.status) {
return dataB.createdAt.getTime() - dataA.createdAt.getTime();
}
@@ -120,13 +120,7 @@ export default function MediaServerWidget({
>
<Tooltip label={t(`titles.stats.${stat.name}`)}>
<Card p={0} radius={board.itemRadius} className={classes.card}>
<Group
className="mediaRequests-stats-stat-stack"
justify="center"
align="center"
gap="xs"
w="100%"
>
<Group className="mediaRequests-stats-stat-stack" justify="center" align="center" gap="xs" w="100%">
<stat.icon className="mediaRequests-stats-stat-icon" size={16} />
<Text className="mediaRequests-stats-stat-value" size="md">
{stat.number}
@@ -142,13 +136,7 @@ export default function MediaServerWidget({
<Text className="mediaRequests-stats-users-title" fw="bold" ta="center" size={isTiny ? "xs" : "sm"}>
{t("titles.users.main")} ({t("titles.users.requests")})
</Text>
<Stack
className="mediaRequests-stats-users-wrapper"
flex={1}
w="100%"
gap={4}
style={{ overflow: "hidden" }}
>
<Stack className="mediaRequests-stats-users-wrapper" flex={1} w="100%" gap={4} style={{ overflow: "hidden" }}>
{requestStats.users.slice(0, 10).map((user) => (
<Card
component="a"
@@ -164,13 +152,7 @@ export default function MediaServerWidget({
p="xs"
radius={board.itemRadius}
>
<Group
className="mediaRequests-stats-users-user-group"
h="100%"
p={0}
gap="sm"
justify="space-between"
>
<Group className="mediaRequests-stats-users-user-group" h="100%" p={0} gap="sm" justify="space-between">
<Group gap={4}>
<Tooltip label={user.integration.name}>
<Avatar