perf(icons): updater takes 10 seconds to run with high cpu usage (#5608)

packages/common/src/stopwatch.ts

@@ -41,6 +41,10 @@ export class Stopwatch {
     return dayjs().millisecond(this.startTime).fromNow(true);
   }
 
+  getElapsedInFormattedMilliseconds() {
+    return `${this.getElapsedInMilliseconds().toFixed(0)}ms`;
+  }
+
   getElapsedInMilliseconds() {
     return performance.now() - this.startTime;
   }

packages/cron-jobs/src/jobs/icons-updater.ts

@@ -1,9 +1,12 @@
+import type { MySqlRawQueryResult } from "drizzle-orm/mysql2";
+import type { QueryResult } from "pg";
+
 import { createId, splitToNChunks, Stopwatch } from "@homarr/common";
 import { env } from "@homarr/common/env";
 import { createLogger } from "@homarr/core/infrastructure/logs";
 import { EVERY_WEEK } from "@homarr/cron-jobs-core/expressions";
 import type { InferInsertModel } from "@homarr/db";
-import { db, handleTransactionsAsync, inArray, sql } from "@homarr/db";
+import { and, db, eq, handleTransactionsAsync, inArray, notInArray, or } from "@homarr/db";
 import { iconRepositories, icons } from "@homarr/db/schema";
 import { fetchIconsAsync } from "@homarr/icons";
 
@@ -26,7 +29,7 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, {
   logger.info("Fetched icons from repositories", {
     repositoryCount: repositoryIconGroups.length,
     iconCount: countIcons,
-    duration: stopWatch.getElapsedInHumanWords(),
+    duration: stopWatch.getElapsedInFormattedMilliseconds(),
   });
 
   const databaseIconRepositories = await db.query.iconRepositories.findMany({
@@ -35,7 +38,7 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, {
     },
   });
 
-  const skippedChecksums: `${string}.${string}`[] = [];
+  const skippedChecksums = new Map<string, string[]>();
   let countDeleted = 0;
   let countInserted = 0;
 
@@ -50,6 +53,7 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, {
     if (!repositoryIconGroup.success) {
       continue;
     }
+    const localSkippedChecksums: string[] = [];
 
     const repositoryInDb = databaseIconRepositories.find(
       (dbIconGroup) => dbIconGroup.slug === repositoryIconGroup.slug,
@@ -66,7 +70,7 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, {
 
     for (const icon of repositoryIconGroup.icons) {
       if (dbIconsInRepository.some((dbIcon) => dbIcon.checksum === icon.checksum)) {
-        skippedChecksums.push(`${iconRepositoryId}.${icon.checksum}`);
+        localSkippedChecksums.push(icon.checksum);
         continue;
       }
 
@@ -79,38 +83,28 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, {
       });
       countInserted++;
     }
-  }
 
-  const deadIcons = allDbIcons.filter(
-    (icon) => !skippedChecksums.includes(`${icon.iconRepositoryId}.${icon.checksum}`),
-  );
+    skippedChecksums.set(iconRepositoryId, localSkippedChecksums);
+  }
 
   const deadIconRepositories = databaseIconRepositories.filter(
     (iconRepository) => !repositoryIconGroups.some((group) => group.slug === iconRepository.slug),
   );
 
+  const deadIconsFilter = or(
+    ...[...skippedChecksums.entries()].map(([iconRepositoryId, checksums]) =>
+      and(eq(icons.iconRepositoryId, iconRepositoryId), notInArray(icons.checksum, checksums)),
+    ),
+  );
+
   await handleTransactionsAsync(db, {
     async handleAsync(db, schema) {
       await db.transaction(async (transaction) => {
-        if (newIconRepositories.length >= 1) {
-          await transaction.insert(schema.iconRepositories).values(newIconRepositories);
-        }
+        const result = (await transaction.delete(schema.icons).where(deadIconsFilter)) as
+          | MySqlRawQueryResult
+          | QueryResult;
 
-        if (newIcons.length >= 1) {
-          // We only insert 5000 icons at a time to avoid SQLite limitations
-          for (const chunck of splitToNChunks(newIcons, Math.ceil(newIcons.length / 5000))) {
-            await transaction.insert(schema.icons).values(chunck);
-          }
-        }
-        if (deadIcons.length >= 1) {
-          await transaction.delete(schema.icons).where(
-            inArray(
-              // Combine iconRepositoryId and checksum to allow same icons on different repositories
-              sql`concat(${icons.iconRepositoryId}, '.', ${icons.checksum})`,
-              deadIcons.map((icon) => `${icon.iconRepositoryId}.${icon.checksum}`),
-            ),
-          );
-        }
+        countDeleted += Array.isArray(result) ? result[0].affectedRows : (result.rowCount ?? 0);
 
         if (deadIconRepositories.length >= 1) {
           await transaction.delete(schema.iconRepositories).where(
@@ -121,33 +115,22 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, {
           );
         }
 
-        countDeleted += deadIcons.length;
-      });
-    },
-    handleSync() {
-      db.transaction((transaction) => {
         if (newIconRepositories.length >= 1) {
-          transaction.insert(iconRepositories).values(newIconRepositories).run();
+          await transaction.insert(schema.iconRepositories).values(newIconRepositories);
         }
 
         if (newIcons.length >= 1) {
           // We only insert 5000 icons at a time to avoid SQLite limitations
           for (const chunck of splitToNChunks(newIcons, Math.ceil(newIcons.length / 5000))) {
-            transaction.insert(icons).values(chunck).run();
+            await transaction.insert(schema.icons).values(chunck);
           }
         }
-        if (deadIcons.length >= 1) {
-          transaction
-            .delete(icons)
-            .where(
-              inArray(
-                // Combine iconRepositoryId and checksum to allow same icons on different repositories
-                sql`concat(${icons.iconRepositoryId}, '.', ${icons.checksum})`,
-                deadIcons.map((icon) => `${icon.iconRepositoryId}.${icon.checksum}`),
-              ),
-            )
-            .run();
-        }
+      });
+    },
+    handleSync() {
+      db.transaction((transaction) => {
+        const result = transaction.delete(icons).where(deadIconsFilter).run();
+        countDeleted += result.changes;
 
         if (deadIconRepositories.length >= 1) {
           transaction
@@ -161,13 +144,22 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, {
             .run();
         }
 
-        countDeleted += deadIcons.length;
+        if (newIconRepositories.length >= 1) {
+          transaction.insert(iconRepositories).values(newIconRepositories).run();
+        }
+
+        if (newIcons.length >= 1) {
+          // We only insert 5000 icons at a time to avoid SQLite limitations
+          for (const chunck of splitToNChunks(newIcons, Math.ceil(newIcons.length / 5000))) {
+            transaction.insert(icons).values(chunck).run();
+          }
+        }
       });
     },
   });
 
   logger.info("Updated icons in database", {
-    duration: stopWatch.getElapsedInHumanWords(),
+    duration: stopWatch.getElapsedInFormattedMilliseconds(),
     added: countInserted,
     deleted: countDeleted,
   });

packages/db/package.json

@@ -37,6 +37,8 @@
     "push:sqlite": "pnpm with-env drizzle-kit push --config ./configs/sqlite.config.ts",
     "seed": "pnpm with-env tsx ./migrations/run-seed.ts",
     "studio": "pnpm with-env drizzle-kit studio --config ./configs/sqlite.config.ts",
+    "studio:mysql": "pnpm with-env drizzle-kit studio --config ./configs/mysql.config.ts",
+    "studio:postgresql": "pnpm with-env drizzle-kit studio --config ./configs/postgresql.config.ts",
     "typecheck": "tsc --noEmit",
     "with-env": "dotenv -e ../../.env --"
   },