diff --git a/packages/common/src/stopwatch.ts b/packages/common/src/stopwatch.ts index 33693631c..c58697083 100644 --- a/packages/common/src/stopwatch.ts +++ b/packages/common/src/stopwatch.ts @@ -41,6 +41,10 @@ export class Stopwatch { return dayjs().millisecond(this.startTime).fromNow(true); } + getElapsedInFormattedMilliseconds() { + return `${this.getElapsedInMilliseconds().toFixed(0)}ms`; + } + getElapsedInMilliseconds() { return performance.now() - this.startTime; } diff --git a/packages/cron-jobs/src/jobs/icons-updater.ts b/packages/cron-jobs/src/jobs/icons-updater.ts index b48c26ea6..cc33727b8 100644 --- a/packages/cron-jobs/src/jobs/icons-updater.ts +++ b/packages/cron-jobs/src/jobs/icons-updater.ts @@ -1,9 +1,12 @@ +import type { MySqlRawQueryResult } from "drizzle-orm/mysql2"; +import type { QueryResult } from "pg"; + import { createId, splitToNChunks, Stopwatch } from "@homarr/common"; import { env } from "@homarr/common/env"; import { createLogger } from "@homarr/core/infrastructure/logs"; import { EVERY_WEEK } from "@homarr/cron-jobs-core/expressions"; import type { InferInsertModel } from "@homarr/db"; -import { db, handleTransactionsAsync, inArray, sql } from "@homarr/db"; +import { and, db, eq, handleTransactionsAsync, inArray, notInArray, or } from "@homarr/db"; import { iconRepositories, icons } from "@homarr/db/schema"; import { fetchIconsAsync } from "@homarr/icons"; @@ -26,7 +29,7 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, { logger.info("Fetched icons from repositories", { repositoryCount: repositoryIconGroups.length, iconCount: countIcons, - duration: stopWatch.getElapsedInHumanWords(), + duration: stopWatch.getElapsedInFormattedMilliseconds(), }); const databaseIconRepositories = await db.query.iconRepositories.findMany({ @@ -35,7 +38,7 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, { }, }); - const skippedChecksums: `${string}.${string}`[] = []; + const skippedChecksums = new Map(); let countDeleted = 0; let countInserted = 0; @@ -50,6 +53,7 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, { if (!repositoryIconGroup.success) { continue; } + const localSkippedChecksums: string[] = []; const repositoryInDb = databaseIconRepositories.find( (dbIconGroup) => dbIconGroup.slug === repositoryIconGroup.slug, @@ -66,7 +70,7 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, { for (const icon of repositoryIconGroup.icons) { if (dbIconsInRepository.some((dbIcon) => dbIcon.checksum === icon.checksum)) { - skippedChecksums.push(`${iconRepositoryId}.${icon.checksum}`); + localSkippedChecksums.push(icon.checksum); continue; } @@ -79,38 +83,28 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, { }); countInserted++; } - } - const deadIcons = allDbIcons.filter( - (icon) => !skippedChecksums.includes(`${icon.iconRepositoryId}.${icon.checksum}`), - ); + skippedChecksums.set(iconRepositoryId, localSkippedChecksums); + } const deadIconRepositories = databaseIconRepositories.filter( (iconRepository) => !repositoryIconGroups.some((group) => group.slug === iconRepository.slug), ); + const deadIconsFilter = or( + ...[...skippedChecksums.entries()].map(([iconRepositoryId, checksums]) => + and(eq(icons.iconRepositoryId, iconRepositoryId), notInArray(icons.checksum, checksums)), + ), + ); + await handleTransactionsAsync(db, { async handleAsync(db, schema) { await db.transaction(async (transaction) => { - if (newIconRepositories.length >= 1) { - await transaction.insert(schema.iconRepositories).values(newIconRepositories); - } + const result = (await transaction.delete(schema.icons).where(deadIconsFilter)) as + | MySqlRawQueryResult + | QueryResult; - if (newIcons.length >= 1) { - // We only insert 5000 icons at a time to avoid SQLite limitations - for (const chunck of splitToNChunks(newIcons, Math.ceil(newIcons.length / 5000))) { - await transaction.insert(schema.icons).values(chunck); - } - } - if (deadIcons.length >= 1) { - await transaction.delete(schema.icons).where( - inArray( - // Combine iconRepositoryId and checksum to allow same icons on different repositories - sql`concat(${icons.iconRepositoryId}, '.', ${icons.checksum})`, - deadIcons.map((icon) => `${icon.iconRepositoryId}.${icon.checksum}`), - ), - ); - } + countDeleted += Array.isArray(result) ? result[0].affectedRows : (result.rowCount ?? 0); if (deadIconRepositories.length >= 1) { await transaction.delete(schema.iconRepositories).where( @@ -121,33 +115,22 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, { ); } - countDeleted += deadIcons.length; - }); - }, - handleSync() { - db.transaction((transaction) => { if (newIconRepositories.length >= 1) { - transaction.insert(iconRepositories).values(newIconRepositories).run(); + await transaction.insert(schema.iconRepositories).values(newIconRepositories); } if (newIcons.length >= 1) { // We only insert 5000 icons at a time to avoid SQLite limitations for (const chunck of splitToNChunks(newIcons, Math.ceil(newIcons.length / 5000))) { - transaction.insert(icons).values(chunck).run(); + await transaction.insert(schema.icons).values(chunck); } } - if (deadIcons.length >= 1) { - transaction - .delete(icons) - .where( - inArray( - // Combine iconRepositoryId and checksum to allow same icons on different repositories - sql`concat(${icons.iconRepositoryId}, '.', ${icons.checksum})`, - deadIcons.map((icon) => `${icon.iconRepositoryId}.${icon.checksum}`), - ), - ) - .run(); - } + }); + }, + handleSync() { + db.transaction((transaction) => { + const result = transaction.delete(icons).where(deadIconsFilter).run(); + countDeleted += result.changes; if (deadIconRepositories.length >= 1) { transaction @@ -161,13 +144,22 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, { .run(); } - countDeleted += deadIcons.length; + if (newIconRepositories.length >= 1) { + transaction.insert(iconRepositories).values(newIconRepositories).run(); + } + + if (newIcons.length >= 1) { + // We only insert 5000 icons at a time to avoid SQLite limitations + for (const chunck of splitToNChunks(newIcons, Math.ceil(newIcons.length / 5000))) { + transaction.insert(icons).values(chunck).run(); + } + } }); }, }); logger.info("Updated icons in database", { - duration: stopWatch.getElapsedInHumanWords(), + duration: stopWatch.getElapsedInFormattedMilliseconds(), added: countInserted, deleted: countDeleted, }); diff --git a/packages/db/package.json b/packages/db/package.json index 2dd2b1070..6e6c4851d 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -37,6 +37,8 @@ "push:sqlite": "pnpm with-env drizzle-kit push --config ./configs/sqlite.config.ts", "seed": "pnpm with-env tsx ./migrations/run-seed.ts", "studio": "pnpm with-env drizzle-kit studio --config ./configs/sqlite.config.ts", + "studio:mysql": "pnpm with-env drizzle-kit studio --config ./configs/mysql.config.ts", + "studio:postgresql": "pnpm with-env drizzle-kit studio --config ./configs/postgresql.config.ts", "typecheck": "tsc --noEmit", "with-env": "dotenv -e ../../.env --" },