deDuplicateAll and keepAll endpoints

open-api/immich-openapi-specs.json

@@ -4645,6 +4645,60 @@
         "x-immich-state": "Stable"
       }
     },
+    "/duplicates/de-duplicate-all": {
+      "delete": {
+        "operationId": "deDuplicateAll",
+        "parameters": [],
+        "responses": {
+          "204": {
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Duplicates"
+        ],
+        "x-immich-permission": "duplicate.delete",
+        "description": "This endpoint requires the `duplicate.delete` permission."
+      }
+    },
+    "/duplicates/keep-all": {
+      "delete": {
+        "operationId": "keepAll",
+        "parameters": [],
+        "responses": {
+          "204": {
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Duplicates"
+        ],
+        "x-immich-permission": "duplicate.delete",
+        "description": "This endpoint requires the `duplicate.delete` permission."
+      }
+    },
     "/duplicates/{id}": {
       "delete": {
         "description": "Delete a single duplicate asset specified by its ID.",

open-api/typescript-sdk/src/fetch-client.ts

@@ -2885,6 +2885,24 @@ export function getAssetDuplicates({ page, size }: {
 /**
  * Delete a duplicate
  */
+export function deDuplicateAll(opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchText("/duplicates/de-duplicate-all", {
+        ...opts,
+        method: "DELETE"
+    }));
+}
+/**
+ * This endpoint requires the `duplicate.delete` permission.
+ */
+export function keepAll(opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchText("/duplicates/keep-all", {
+        ...opts,
+        method: "DELETE"
+    }));
+}
+/**
+ * This endpoint requires the `duplicate.delete` permission.
+ */
 export function deleteDuplicate({ id }: {
     id: string;
 }, opts?: Oazapfts.RequestOpts) {

pnpm-lock.yaml

@@ -469,6 +469,9 @@ importers:
       lodash:
         specifier: ^4.17.21
         version: 4.17.21
+      lodash-es:
+        specifier: ^4.17.21
+        version: 4.17.21
       luxon:
         specifier: ^3.4.2
         version: 3.7.2
@@ -604,7 +607,10 @@ importers:
         version: 9.0.10
       '@types/lodash':
         specifier: ^4.14.197
-        version: 4.17.21
+        version: 4.17.20
+      '@types/lodash-es':
+        specifier: ^4.17.12
+        version: 4.17.12
       '@types/luxon':
         specifier: ^3.6.2
         version: 3.7.1

server/package.json

@@ -84,6 +84,7 @@
     "kysely": "0.28.2",
     "kysely-postgres-js": "^3.0.0",
     "lodash": "^4.17.21",
+    "lodash-es": "^4.17.21",
     "luxon": "^3.4.2",
     "mnemonist": "^0.40.3",
     "multer": "^2.0.2",
@@ -131,6 +132,7 @@
     "@types/jsonwebtoken": "^9.0.10",
     "@types/js-yaml": "^4.0.9",
     "@types/lodash": "^4.14.197",
+    "@types/lodash-es": "^4.17.12",
     "@types/luxon": "^3.6.2",
     "@types/mock-fs": "^4.13.1",
     "@types/multer": "^2.0.0",

server/src/controllers/duplicate.controller.ts

@@ -1,4 +1,4 @@
-import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, Query } from '@nestjs/common';
+import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, Post, Query } from '@nestjs/common';
 import { ApiQuery, ApiTags } from '@nestjs/swagger';
 import { Endpoint, HistoryBuilder } from 'src/decorators';
 import { BulkIdsDto } from 'src/dtos/asset-ids.response.dto';
@@ -28,6 +28,20 @@ export class DuplicateController {
     return this.service.getDuplicates(auth, page, size);
   }
 
+  @Delete('/de-duplicate-all')
+  @Authenticated({ permission: Permission.DuplicateDelete })
+  @HttpCode(HttpStatus.NO_CONTENT)
+  deDuplicateAll(@Auth() auth: AuthDto) {
+    return this.service.deDuplicateAll(auth);
+  }
+
+  @Delete('/keep-all')
+  @Authenticated({ permission: Permission.DuplicateDelete })
+  @HttpCode(HttpStatus.NO_CONTENT)
+  keepAll(@Auth() auth: AuthDto) {
+    return this.service.keepAll(auth);
+  }
+
   @Delete()
   @Authenticated({ permission: Permission.DuplicateDelete })
   @HttpCode(HttpStatus.NO_CONTENT)

server/src/services/duplicate.service.ts

@@ -1,18 +1,22 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
 import { OnJob } from 'src/decorators';
 import { BulkIdsDto } from 'src/dtos/asset-ids.response.dto';
 import { mapAsset } from 'src/dtos/asset-response.dto';
 import { AuthDto } from 'src/dtos/auth.dto';
 import { DuplicateResponseDto } from 'src/dtos/duplicate.dto';
-import { AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum';
+import { AssetStatus, AssetVisibility, JobName, JobStatus, Permission, QueueName } from 'src/enum';
 import { AssetDuplicateResult } from 'src/repositories/search.repository';
+import { AssetService } from 'src/services/asset.service';
 import { BaseService } from 'src/services/base.service';
 import { JobItem, JobOf } from 'src/types';
+import { suggestDuplicate } from 'src/utils/duplicate-utils';
 import { isDuplicateDetectionEnabled } from 'src/utils/misc';
 
 @Injectable()
 export class DuplicateService extends BaseService {
+  @Inject() private assetService!: AssetService;
+
   async getDuplicates(auth: AuthDto, page = 1, size = 20): Promise<DuplicateResponseDto> {
     const { items, totalItems } = await this.duplicateRepository.getAll(auth.user.id, page, size);
 
@@ -23,12 +27,12 @@ export class DuplicateService extends BaseService {
 
     const totalPages = Math.ceil(totalItems / size);
     const hasNextPage = page < totalPages;
-    
+
     return {
       items: duplicates,
       totalItems,
       totalPages,
-      hasNextPage
+      hasNextPage,
     };
   }
 
@@ -40,6 +44,52 @@ export class DuplicateService extends BaseService {
     await this.duplicateRepository.deleteAll(auth.user.id, dto.ids);
   }
 
+  async deDuplicateAll(auth: AuthDto) {
+    let page = 1;
+    const size = 100;
+    let hasNextPage = true;
+
+    while (hasNextPage) {
+      const duplicates = await this.getDuplicates(auth, page, size);
+
+      const idsToKeep = duplicates.items.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id);
+      const idsToDelete = duplicates.items.flatMap((group, i) =>
+        group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]),
+      );
+
+      // This is duplicated from asset.service - deleteAll()
+      await this.requireAccess({ auth, permission: Permission.AssetDelete, ids: idsToDelete });
+      await this.assetRepository.updateAll(idsToDelete, {
+        deletedAt: new Date(),
+        status: false ? AssetStatus.Deleted : AssetStatus.Trashed,
+      });
+      await this.eventRepository.emit(false ? 'AssetDeleteAll' : 'AssetTrashAll', {
+        assetIds: idsToDelete,
+        userId: auth.user.id,
+      });
+
+      hasNextPage = duplicates.hasNextPage;
+      page++;
+    }
+  }
+
+  async keepAll(auth: AuthDto) {
+    let page = 1;
+    const size = 100;
+    let hasNextPage = true;
+
+    while (hasNextPage) {
+      const duplicates = await this.getDuplicates(auth, page, size);
+
+      const idsToDelete = duplicates.items.map(({ duplicateId }) => duplicateId);
+
+      await this.deleteAll(auth, { ids: idsToDelete });
+
+      hasNextPage = duplicates.hasNextPage;
+      page++;
+    }
+  }
+
   @OnJob({ name: JobName.AssetDetectDuplicatesQueueAll, queue: QueueName.DuplicateDetection })
   async handleQueueSearchDuplicates({ force }: JobOf<JobName.AssetDetectDuplicatesQueueAll>): Promise<JobStatus> {
     const { machineLearning } = await this.getConfig({ withCache: false });

server/src/utils/duplicate-utils.spec.ts

@@ -0,0 +1,38 @@
+import { AssetResponseDto } from "src/dtos/asset-response.dto";
+import { suggestDuplicate } from "src/utils/duplicate-utils";
+
+
+describe('choosing a duplicate', () => {
+  it('picks the asset with the largest file size', () => {
+    const assets = [
+      { exifInfo: { fileSizeInByte: 300 } },
+      { exifInfo: { fileSizeInByte: 200 } },
+      { exifInfo: { fileSizeInByte: 100 } },
+    ];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+
+  it('picks the asset with the most exif data if multiple assets have the same file size', () => {
+    const assets = [
+      { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1 } },
+      { exifInfo: { fileSizeInByte: 200, rating: 5 } },
+      { exifInfo: { fileSizeInByte: 100, rating: 5 } },
+    ];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+
+  it('returns undefined for an empty array', () => {
+    const assets: AssetResponseDto[] = [];
+    expect(suggestDuplicate(assets)).toBeUndefined();
+  });
+
+  it('handles assets with no exifInfo', () => {
+    const assets = [{ exifInfo: { fileSizeInByte: 200 } }, {}];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+
+  it('handles assets with exifInfo but no fileSizeInByte', () => {
+    const assets = [{ exifInfo: { rating: 5, fNumber: 1 } }, { exifInfo: { rating: 5 } }];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+});

server/src/utils/duplicate-utils.ts

@@ -0,0 +1,30 @@
+import { sortBy } from "lodash";
+import { AssetResponseDto } from "src/dtos/asset-response.dto";
+import { getExifCount } from "src/utils/exif-utils";
+
+/**
+ * Suggests the best duplicate asset to keep from a list of duplicates.
+ *
+ * The best asset is determined by the following criteria:
+ *  - Largest image file size in bytes
+ *  - Largest count of exif data
+ *
+ * @param assets List of duplicate assets
+ * @returns The best asset to keepweb/src/lib/utils/duplicate-utils.spec.ts
+ */
+export const suggestDuplicate = (assets: AssetResponseDto[]): AssetResponseDto | undefined => {
+  let duplicateAssets = sortBy(assets, (asset) => asset.exifInfo?.fileSizeInByte ?? 0);
+
+  // Update the list to only include assets with the largest file size
+  duplicateAssets = duplicateAssets.filter(
+    (asset) => asset.exifInfo?.fileSizeInByte === duplicateAssets.at(-1)?.exifInfo?.fileSizeInByte,
+  );
+
+  // If there are multiple assets with the same file size, sort the list by the count of exif data
+  if (duplicateAssets.length >= 2) {
+    duplicateAssets = sortBy(duplicateAssets, getExifCount);
+  }
+
+  // Return the last asset in the list
+  return duplicateAssets.pop();
+};

server/src/utils/exif-utils.spec.ts

@@ -0,0 +1,30 @@
+import { AssetResponseDto } from "src/dtos/asset-response.dto";
+import { getExifCount } from "src/utils/exif-utils";
+
+
+describe('getting the exif count', () => {
+  it('returns 0 when exifInfo is undefined', () => {
+    const asset = {};
+    expect(getExifCount(asset as AssetResponseDto)).toBe(0);
+  });
+
+  it('returns 0 when exifInfo is empty', () => {
+    const asset = { exifInfo: {} };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(0);
+  });
+
+  it('returns the correct count of non-null exifInfo properties', () => {
+    const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: null } };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(2);
+  });
+
+  it('ignores null, undefined and empty properties in exifInfo', () => {
+    const asset = { exifInfo: { fileSizeInByte: 200, rating: null, fNumber: undefined, description: '' } };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(1);
+  });
+
+  it('returns the correct count when all exifInfo properties are non-null', () => {
+    const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1, description: 'test' } };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(4);
+  });
+});

server/src/utils/exif-utils.ts

@@ -0,0 +1,6 @@
+import { AssetResponseDto } from "src/dtos/asset-response.dto";
+
+
+export const getExifCount = (asset: AssetResponseDto) => {
+  return Object.values(asset.exifInfo ?? {}).filter(Boolean).length;
+};

web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte

@@ -11,11 +11,10 @@
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
   import { locale } from '$lib/stores/preferences.store';
   import { stackAssets } from '$lib/utils/asset-utils';
-  import { suggestDuplicate } from '$lib/utils/duplicate-utils';
   import { handleError } from '$lib/utils/handle-error';
   import type { AssetResponseDto } from '@immich/sdk';
-  import { deleteAssets, deleteDuplicates, updateAssets } from '@immich/sdk';
   import { Button, HStack, IconButton, modalManager, Text, toastManager } from '@immich/ui';
+  import { deDuplicateAll, deleteAssets, keepAll, updateAssets } from '@immich/sdk';
   import {
     mdiCheckOutline,
     mdiChevronLeft,
@@ -56,11 +55,14 @@
     ],
   };
 
-  let duplicates = $state(data.duplicates);
+  let duplicatesRes = $state(data.duplicatesRes);
+  // let duplicates = $state(data.duplicates);
+  let duplicate = $state(data.duplicate);
+
   const { isViewing: showAssetViewer } = assetViewingStore;
 
   const correctDuplicatesIndex = (index: number) => {
-    return Math.max(0, Math.min(index, duplicates.length - 1));
+    return Math.max(0, Math.min(index, duplicatesRes.totalItems - 1));
   };
 
   let duplicatesIndex = $derived(
@@ -71,7 +73,7 @@
     })(),
   );
 
-  let hasDuplicates = $derived(duplicates.length > 0);
+  let hasDuplicates = $derived(duplicatesRes.totalItems > 0);
   const withConfirmation = async (callback: () => Promise<void>, prompt?: string, confirmText?: string) => {
     if (prompt && confirmText) {
       const isConfirmed = await modalManager.showDialog({ prompt, confirmText });
@@ -104,7 +106,7 @@
         await deleteAssets({ assetBulkDeleteDto: { ids: trashIds, force: !featureFlagsManager.value.trash } });
         await updateAssets({ assetBulkUpdateDto: { ids: duplicateAssetIds, duplicateId: null } });
 
-        duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId);
+        // duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId);
 
         deletedNotification(trashIds.length);
         await correctDuplicatesIndexAndGo(duplicatesIndex);
@@ -118,39 +120,24 @@
     await stackAssets(assets, false);
     const duplicateAssetIds = assets.map((asset) => asset.id);
     await updateAssets({ assetBulkUpdateDto: { ids: duplicateAssetIds, duplicateId: null } });
-    duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId);
+    // duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId);
     await correctDuplicatesIndexAndGo(duplicatesIndex);
   };
 
-  const handleDeduplicateAll = async () => {
-    const idsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id);
-    const idsToDelete = duplicates.flatMap((group, i) =>
-      group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]),
-    );
-
+  const handleDeduplicateAll = () => {
     let prompt, confirmText;
-    if (featureFlagsManager.value.trash) {
-      prompt = $t('bulk_trash_duplicates_confirmation', { values: { count: idsToDelete.length } });
+    if ($featureFlags.trash) {
+      prompt = $t('bulk_trash_duplicates_confirmation', { values: { count: 1 } });
       confirmText = $t('confirm');
     } else {
-      prompt = $t('bulk_delete_duplicates_confirmation', { values: { count: idsToDelete.length } });
+      prompt = $t('bulk_delete_duplicates_confirmation', { values: { count: 1 } });
       confirmText = $t('permanently_delete');
     }
 
     return withConfirmation(
       async () => {
-        await deleteAssets({ assetBulkDeleteDto: { ids: idsToDelete, force: !featureFlagsManager.value.trash } });
-        await updateAssets({
-          assetBulkUpdateDto: {
-            ids: [...idsToDelete, ...idsToKeep.filter((id): id is string => !!id)],
-            duplicateId: null,
-          },
-        });
-
-        duplicates = [];
-
-        deletedNotification(idsToDelete.length);
-
+        await deDuplicateAll();
+        deletedNotification(1);
         page.url.searchParams.delete('index');
         await goto(`${AppRoute.DUPLICATES}`);
       },
@@ -159,19 +146,16 @@
     );
   };
 
-  const handleKeepAll = async () => {
-    const ids = duplicates.map(({ duplicateId }) => duplicateId);
+  const handleKeepAll = () => {
     return withConfirmation(
       async () => {
-        await deleteDuplicates({ bulkIdsDto: { ids } });
-
-        duplicates = [];
+        await keepAll();
 
         toastManager.success($t('resolved_all_duplicates'));
         page.url.searchParams.delete('index');
         await goto(`${AppRoute.DUPLICATES}`);
       },
-      $t('bulk_keep_duplicates_confirmation', { values: { count: ids.length } }),
+      $t('bulk_keep_duplicates_confirmation', { values: { count: 1 } }),
       $t('confirm'),
     );
   };
@@ -189,7 +173,7 @@
     await handlePrevious();
   };
   const handleNext = async () => {
-    await correctDuplicatesIndexAndGo(Math.min(duplicatesIndex + 1, duplicates.length - 1));
+    await correctDuplicatesIndexAndGo(Math.min(duplicatesIndex + 1, duplicatesRes.totalItems - 1));
   };
   const handleNextShortcut = async () => {
     if ($showAssetViewer) {
@@ -198,11 +182,14 @@
     await handleNext();
   };
   const handleLast = async () => {
-    await correctDuplicatesIndexAndGo(duplicates.length - 1);
+    await correctDuplicatesIndexAndGo(duplicatesRes.totalItems - 1);
   };
   const correctDuplicatesIndexAndGo = async (index: number) => {
     page.url.searchParams.set('index', correctDuplicatesIndex(index).toString());
     await goto(`${AppRoute.DUPLICATES}?${page.url.searchParams.toString()}`);
+    const result = await data.loadDuplicates(index + 1, 1);
+    duplicate = result.items[0];
+    duplicatesRes = result;
   };
 </script>
 
@@ -213,7 +200,7 @@
   ]}
 />
 
-<UserPageLayout title={data.meta.title + ` (${duplicates.length.toLocaleString($locale)})`} scrollbar={true}>
+<UserPageLayout title={data.meta.title + ` (${duplicatesRes.totalItems.toLocaleString($locale)})`} scrollbar={true}>
   {#snippet buttons()}
     <HStack gap={0}>
       <Button
@@ -248,8 +235,8 @@
     </HStack>
   {/snippet}
 
-  <div class="">
-    {#if duplicates && duplicates.length > 0}
+  <div>
+    {#if duplicate && duplicatesRes.totalItems > 0}
       <div class="flex items-center mb-2">
         <div class="text-sm dark:text-white">
           <p>{$t('duplicates_description')}</p>
@@ -265,12 +252,12 @@
         />
       </div>
 
-      {#key duplicates[duplicatesIndex].duplicateId}
+      {#key duplicate.duplicateId}
         <DuplicatesCompareControl
-          assets={duplicates[duplicatesIndex].assets}
+          assets={duplicate.assets}
           onResolve={(duplicateAssetIds, trashIds) =>
-            handleResolve(duplicates[duplicatesIndex].duplicateId, duplicateAssetIds, trashIds)}
-          onStack={(assets) => handleStack(duplicates[duplicatesIndex].duplicateId, assets)}
+            handleResolve(duplicate.duplicateId, duplicateAssetIds, trashIds)}
+          onStack={(assets) => handleStack(duplicate.duplicateId, assets)}
         />
         <div class="max-w-5xl mx-auto mb-16">
           <div class="flex mb-4 sm:px-6 w-full place-content-center justify-between items-center place-items-center">
@@ -296,9 +283,7 @@
                 {$t('previous')}
               </Button>
             </div>
-            <p class="border px-3 md:px-6 py-1 dark:bg-subtle rounded-lg text-xs md:text-sm">
-              {duplicatesIndex + 1} / {duplicates.length.toLocaleString($locale)}
-            </p>
+            <p>{duplicatesIndex + 1}/{duplicatesRes.totalItems.toLocaleString($locale)}</p>
             <div class="flex text-xs text-black">
               <Button
                 size="small"
@@ -306,7 +291,7 @@
                 color="primary"
                 class="flex place-items-center rounded-s-full gap-2 px-2 sm:px-4"
                 onclick={handleNext}
-                disabled={duplicatesIndex === duplicates.length - 1}
+                disabled={duplicatesIndex === duplicatesRes.totalItems - 1}
               >
                 {$t('next')}
               </Button>
@@ -316,7 +301,7 @@
                 color="primary"
                 class="flex place-items-center rounded-e-full gap-2 px-2 sm:px-4"
                 onclick={handleLast}
-                disabled={duplicatesIndex === duplicates.length - 1}
+                disabled={duplicatesIndex === duplicatesRes.totalItems - 1}
               >
                 {$t('last')}
               </Button>

web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.ts

@@ -7,12 +7,24 @@ import type { PageLoad } from './$types';
 export const load = (async ({ params, url }) => {
   await authenticate(url);
   const asset = await getAssetInfoFromParam(params);
-  const duplicates = await getAssetDuplicates();
   const $t = await getFormatter();
 
+  const indexParam = url.searchParams.get('index') ?? '0';
+  const parsedIndex = Number.parseInt(indexParam, 10);
+  
+  const duplicates = await getAssetDuplicates({ page: parsedIndex + 1, size: 1 });
+  const duplicate = duplicates.items[0];
+
+  const loadDuplicates = async (newPage: number, newSize: number) => {
+    return await getAssetDuplicates({ page: newPage, size: newSize });
+  };
+
   return {
     asset,
-    duplicates,
+    duplicatesRes: duplicates,
+    duplicates: duplicates.items,
+    duplicate,
+    loadDuplicates,
     meta: {
       title: $t('duplicates'),
     },