Merge a9a05671f4 into 6d499c782a

mobile/openapi/README.md

@@ -143,9 +143,13 @@ Class | Method | HTTP request | Description
 *DeprecatedApi* | [**runQueueCommandLegacy**](doc//DeprecatedApi.md#runqueuecommandlegacy) | **PUT** /jobs/{name} | Run jobs
 *DownloadApi* | [**downloadArchive**](doc//DownloadApi.md#downloadarchive) | **POST** /download/archive | Download asset archive
 *DownloadApi* | [**getDownloadInfo**](doc//DownloadApi.md#getdownloadinfo) | **POST** /download/info | Retrieve download information
+*DuplicatesApi* | [**countDeDuplicateAll**](doc//DuplicatesApi.md#countdeduplicateall) | **GET** /duplicates/de-duplicate-all/count | 
+*DuplicatesApi* | [**countKeepAll**](doc//DuplicatesApi.md#countkeepall) | **GET** /duplicates/keep-all/count | 
+*DuplicatesApi* | [**deDuplicateAll**](doc//DuplicatesApi.md#deduplicateall) | **DELETE** /duplicates/de-duplicate-all | 
 *DuplicatesApi* | [**deleteDuplicate**](doc//DuplicatesApi.md#deleteduplicate) | **DELETE** /duplicates/{id} | Delete a duplicate
 *DuplicatesApi* | [**deleteDuplicates**](doc//DuplicatesApi.md#deleteduplicates) | **DELETE** /duplicates | Delete duplicates
 *DuplicatesApi* | [**getAssetDuplicates**](doc//DuplicatesApi.md#getassetduplicates) | **GET** /duplicates | Retrieve duplicates
+*DuplicatesApi* | [**keepAll**](doc//DuplicatesApi.md#keepall) | **DELETE** /duplicates/keep-all | 
 *FacesApi* | [**createFace**](doc//FacesApi.md#createface) | **POST** /faces | Create a face
 *FacesApi* | [**deleteFace**](doc//FacesApi.md#deleteface) | **DELETE** /faces/{id} | Delete a face
 *FacesApi* | [**getFaces**](doc//FacesApi.md#getfaces) | **GET** /faces | Retrieve faces for asset
@@ -393,6 +397,7 @@ Class | Method | HTTP request | Description
  - [DownloadResponseDto](doc//DownloadResponseDto.md)
  - [DownloadUpdate](doc//DownloadUpdate.md)
  - [DuplicateDetectionConfig](doc//DuplicateDetectionConfig.md)
+ - [DuplicateItem](doc//DuplicateItem.md)
  - [DuplicateResponseDto](doc//DuplicateResponseDto.md)
  - [EmailNotificationsResponse](doc//EmailNotificationsResponse.md)
  - [EmailNotificationsUpdate](doc//EmailNotificationsUpdate.md)

mobile/openapi/lib/api.dart

@@ -145,6 +145,7 @@ part 'model/download_response.dart';
 part 'model/download_response_dto.dart';
 part 'model/download_update.dart';
 part 'model/duplicate_detection_config.dart';
+part 'model/duplicate_item.dart';
 part 'model/duplicate_response_dto.dart';
 part 'model/email_notifications_response.dart';
 part 'model/email_notifications_update.dart';

mobile/openapi/lib/api/duplicates_api.dart

@@ -16,6 +16,121 @@ class DuplicatesApi {
 
   final ApiClient apiClient;
 
+  /// Performs an HTTP 'GET /duplicates/de-duplicate-all/count' operation and returns the [Response].
+  Future<Response> countDeDuplicateAllWithHttpInfo() async {
+    // ignore: prefer_const_declarations
+    final apiPath = r'/duplicates/de-duplicate-all/count';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      apiPath,
+      'GET',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  Future<num?> countDeDuplicateAll() async {
+    final response = await countDeDuplicateAllWithHttpInfo();
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+    // When a remote server returns no body with a status of 204, we shall not decode it.
+    // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
+    // FormatException when trying to decode an empty string.
+    if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
+      return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'num',) as num;
+    
+    }
+    return null;
+  }
+
+  /// Performs an HTTP 'GET /duplicates/keep-all/count' operation and returns the [Response].
+  Future<Response> countKeepAllWithHttpInfo() async {
+    // ignore: prefer_const_declarations
+    final apiPath = r'/duplicates/keep-all/count';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      apiPath,
+      'GET',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  Future<num?> countKeepAll() async {
+    final response = await countKeepAllWithHttpInfo();
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+    // When a remote server returns no body with a status of 204, we shall not decode it.
+    // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
+    // FormatException when trying to decode an empty string.
+    if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
+      return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'num',) as num;
+    
+    }
+    return null;
+  }
+
+  /// Performs an HTTP 'DELETE /duplicates/de-duplicate-all' operation and returns the [Response].
+  Future<Response> deDuplicateAllWithHttpInfo() async {
+    // ignore: prefer_const_declarations
+    final apiPath = r'/duplicates/de-duplicate-all';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      apiPath,
+      'DELETE',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  Future<void> deDuplicateAll() async {
+    final response = await deDuplicateAllWithHttpInfo();
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+  }
+
   /// Delete a duplicate
   ///
   /// Delete a single duplicate asset specified by its ID.
@@ -118,7 +233,13 @@ class DuplicatesApi {
   /// Retrieve a list of duplicate assets available to the authenticated user.
   ///
   /// Note: This method returns the HTTP [Response].
-  Future<Response> getAssetDuplicatesWithHttpInfo() async {
+  ///
+  /// Parameters:
+  ///
+  /// * [num] page:
+  ///
+  /// * [num] size:
+  Future<Response> getAssetDuplicatesWithHttpInfo({ num? page, num? size, }) async {
     // ignore: prefer_const_declarations
     final apiPath = r'/duplicates';
 
@@ -129,6 +250,13 @@ class DuplicatesApi {
     final headerParams = <String, String>{};
     final formParams = <String, String>{};
 
+    if (page != null) {
+      queryParams.addAll(_queryParams('', 'page', page));
+    }
+    if (size != null) {
+      queryParams.addAll(_queryParams('', 'size', size));
+    }
+
     const contentTypes = <String>[];
 
 
@@ -146,8 +274,14 @@ class DuplicatesApi {
   /// Retrieve duplicates
   ///
   /// Retrieve a list of duplicate assets available to the authenticated user.
-  Future<List<DuplicateResponseDto>?> getAssetDuplicates() async {
-    final response = await getAssetDuplicatesWithHttpInfo();
+  ///
+  /// Parameters:
+  ///
+  /// * [num] page:
+  ///
+  /// * [num] size:
+  Future<DuplicateResponseDto?> getAssetDuplicates({ num? page, num? size, }) async {
+    final response = await getAssetDuplicatesWithHttpInfo( page: page, size: size, );
     if (response.statusCode >= HttpStatus.badRequest) {
       throw ApiException(response.statusCode, await _decodeBodyBytes(response));
     }
@@ -155,12 +289,42 @@ class DuplicatesApi {
     // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
     // FormatException when trying to decode an empty string.
     if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
-      final responseBody = await _decodeBodyBytes(response);
-      return (await apiClient.deserializeAsync(responseBody, 'List<DuplicateResponseDto>') as List)
-        .cast<DuplicateResponseDto>()
-        .toList(growable: false);
-
+      return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'DuplicateResponseDto',) as DuplicateResponseDto;
+    
     }
     return null;
   }
+
+  /// Performs an HTTP 'DELETE /duplicates/keep-all' operation and returns the [Response].
+  Future<Response> keepAllWithHttpInfo() async {
+    // ignore: prefer_const_declarations
+    final apiPath = r'/duplicates/keep-all';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      apiPath,
+      'DELETE',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  Future<void> keepAll() async {
+    final response = await keepAllWithHttpInfo();
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+  }
 }

mobile/openapi/lib/api_client.dart

@@ -338,6 +338,8 @@ class ApiClient {
           return DownloadUpdate.fromJson(value);
         case 'DuplicateDetectionConfig':
           return DuplicateDetectionConfig.fromJson(value);
+        case 'DuplicateItem':
+          return DuplicateItem.fromJson(value);
         case 'DuplicateResponseDto':
           return DuplicateResponseDto.fromJson(value);
         case 'EmailNotificationsResponse':

mobile/openapi/lib/model/duplicate_item.dart

@@ -0,0 +1,107 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.18
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class DuplicateItem {
+  /// Returns a new [DuplicateItem] instance.
+  DuplicateItem({
+    this.assets = const [],
+    required this.duplicateId,
+  });
+
+  List<AssetResponseDto> assets;
+
+  String duplicateId;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is DuplicateItem &&
+    _deepEquality.equals(other.assets, assets) &&
+    other.duplicateId == duplicateId;
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (assets.hashCode) +
+    (duplicateId.hashCode);
+
+  @override
+  String toString() => 'DuplicateItem[assets=$assets, duplicateId=$duplicateId]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'assets'] = this.assets;
+      json[r'duplicateId'] = this.duplicateId;
+    return json;
+  }
+
+  /// Returns a new [DuplicateItem] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static DuplicateItem? fromJson(dynamic value) {
+    upgradeDto(value, "DuplicateItem");
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return DuplicateItem(
+        assets: AssetResponseDto.listFromJson(json[r'assets']),
+        duplicateId: mapValueOfType<String>(json, r'duplicateId')!,
+      );
+    }
+    return null;
+  }
+
+  static List<DuplicateItem> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <DuplicateItem>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = DuplicateItem.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, DuplicateItem> mapFromJson(dynamic json) {
+    final map = <String, DuplicateItem>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = DuplicateItem.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of DuplicateItem-objects as value to a dart map
+  static Map<String, List<DuplicateItem>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<DuplicateItem>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = DuplicateItem.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'assets',
+    'duplicateId',
+  };
+}
+

mobile/openapi/lib/model/duplicate_response_dto.dart

@@ -13,32 +13,44 @@ part of openapi.api;
 class DuplicateResponseDto {
   /// Returns a new [DuplicateResponseDto] instance.
   DuplicateResponseDto({
-    this.assets = const [],
-    required this.duplicateId,
+    required this.hasNextPage,
+    this.items = const [],
+    required this.totalItems,
+    required this.totalPages,
   });
 
-  List<AssetResponseDto> assets;
+  bool hasNextPage;
 
-  String duplicateId;
+  List<DuplicateItem> items;
+
+  num totalItems;
+
+  num totalPages;
 
   @override
   bool operator ==(Object other) => identical(this, other) || other is DuplicateResponseDto &&
-    _deepEquality.equals(other.assets, assets) &&
-    other.duplicateId == duplicateId;
+    other.hasNextPage == hasNextPage &&
+    _deepEquality.equals(other.items, items) &&
+    other.totalItems == totalItems &&
+    other.totalPages == totalPages;
 
   @override
   int get hashCode =>
     // ignore: unnecessary_parenthesis
-    (assets.hashCode) +
-    (duplicateId.hashCode);
+    (hasNextPage.hashCode) +
+    (items.hashCode) +
+    (totalItems.hashCode) +
+    (totalPages.hashCode);
 
   @override
-  String toString() => 'DuplicateResponseDto[assets=$assets, duplicateId=$duplicateId]';
+  String toString() => 'DuplicateResponseDto[hasNextPage=$hasNextPage, items=$items, totalItems=$totalItems, totalPages=$totalPages]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
-      json[r'assets'] = this.assets;
-      json[r'duplicateId'] = this.duplicateId;
+      json[r'hasNextPage'] = this.hasNextPage;
+      json[r'items'] = this.items;
+      json[r'totalItems'] = this.totalItems;
+      json[r'totalPages'] = this.totalPages;
     return json;
   }
 
@@ -51,8 +63,10 @@ class DuplicateResponseDto {
       final json = value.cast<String, dynamic>();
 
       return DuplicateResponseDto(
-        assets: AssetResponseDto.listFromJson(json[r'assets']),
-        duplicateId: mapValueOfType<String>(json, r'duplicateId')!,
+        hasNextPage: mapValueOfType<bool>(json, r'hasNextPage')!,
+        items: DuplicateItem.listFromJson(json[r'items']),
+        totalItems: num.parse('${json[r'totalItems']}'),
+        totalPages: num.parse('${json[r'totalPages']}'),
       );
     }
     return null;
@@ -100,8 +114,10 @@ class DuplicateResponseDto {
 
   /// The list of required keys that must be present in a JSON.
   static const requiredKeys = <String>{
-    'assets',
-    'duplicateId',
+    'hasNextPage',
+    'items',
+    'totalItems',
+    'totalPages',
   };
 }
 

open-api/immich-openapi-specs.json

@@ -4580,16 +4580,32 @@
       "get": {
         "description": "Retrieve a list of duplicate assets available to the authenticated user.",
         "operationId": "getAssetDuplicates",
-        "parameters": [],
+        "parameters": [
+          {
+            "name": "page",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "example": 1,
+              "type": "number"
+            }
+          },
+          {
+            "name": "size",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "example": 20,
+              "type": "number"
+            }
+          }
+        ],
         "responses": {
           "200": {
             "content": {
               "application/json": {
                 "schema": {
-                  "items": {
-                    "$ref": "#/components/schemas/DuplicateResponseDto"
-                  },
-                  "type": "array"
+                  "$ref": "#/components/schemas/DuplicateResponseDto"
                 }
               }
             },
@@ -4629,6 +4645,124 @@
         "x-immich-state": "Stable"
       }
     },
+    "/duplicates/de-duplicate-all": {
+      "delete": {
+        "operationId": "deDuplicateAll",
+        "parameters": [],
+        "responses": {
+          "204": {
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Duplicates"
+        ],
+        "x-immich-permission": "duplicate.delete"
+      }
+    },
+    "/duplicates/de-duplicate-all/count": {
+      "get": {
+        "operationId": "countDeDuplicateAll",
+        "parameters": [],
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "number"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Duplicates"
+        ],
+        "x-immich-permission": "duplicate.read"
+      }
+    },
+    "/duplicates/keep-all": {
+      "delete": {
+        "operationId": "keepAll",
+        "parameters": [],
+        "responses": {
+          "204": {
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Duplicates"
+        ],
+        "x-immich-permission": "duplicate.delete"
+      }
+    },
+    "/duplicates/keep-all/count": {
+      "get": {
+        "operationId": "countKeepAll",
+        "parameters": [],
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "number"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Duplicates"
+        ],
+        "x-immich-permission": "duplicate.read"
+      }
+    },
     "/duplicates/{id}": {
       "delete": {
         "description": "Delete a single duplicate asset specified by its ID.",
@@ -16339,7 +16473,7 @@
         ],
         "type": "object"
       },
-      "DuplicateResponseDto": {
+      "DuplicateItem": {
         "properties": {
           "assets": {
             "items": {
@@ -16357,6 +16491,32 @@
         ],
         "type": "object"
       },
+      "DuplicateResponseDto": {
+        "properties": {
+          "hasNextPage": {
+            "type": "boolean"
+          },
+          "items": {
+            "items": {
+              "$ref": "#/components/schemas/DuplicateItem"
+            },
+            "type": "array"
+          },
+          "totalItems": {
+            "type": "number"
+          },
+          "totalPages": {
+            "type": "number"
+          }
+        },
+        "required": [
+          "hasNextPage",
+          "items",
+          "totalItems",
+          "totalPages"
+        ],
+        "type": "object"
+      },
       "EmailNotificationsResponse": {
         "properties": {
           "albumInvite": {

open-api/typescript-sdk/src/fetch-client.ts

@@ -667,10 +667,16 @@ export type DownloadResponseDto = {
     archives: DownloadArchiveInfo[];
     totalSize: number;
 };
-export type DuplicateResponseDto = {
+export type DuplicateItem = {
     assets: AssetResponseDto[];
     duplicateId: string;
 };
+export type DuplicateResponseDto = {
+    hasNextPage: boolean;
+    items: DuplicateItem[];
+    totalItems: number;
+    totalPages: number;
+};
 export type PersonResponseDto = {
     birthDate: string | null;
     color?: string;
@@ -2862,11 +2868,45 @@ export function deleteDuplicates({ bulkIdsDto }: {
 /**
  * Retrieve duplicates
  */
-export function getAssetDuplicates(opts?: Oazapfts.RequestOpts) {
+export function getAssetDuplicates({ page, size }: {
+    page?: number;
+    size?: number;
+}, opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchJson<{
+        status: 200;
+        data: DuplicateResponseDto;
+    }>(`/duplicates${QS.query(QS.explode({
+        page,
+        size
+    }))}`, {
+        ...opts
+    }));
+}
+export function deDuplicateAll(opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchText("/duplicates/de-duplicate-all", {
+        ...opts,
+        method: "DELETE"
+    }));
+}
+export function countDeDuplicateAll(opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchJson<{
+        status: 200;
+        data: number;
+    }>("/duplicates/de-duplicate-all/count", {
+        ...opts
+    }));
+}
+export function keepAll(opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchText("/duplicates/keep-all", {
+        ...opts,
+        method: "DELETE"
+    }));
+}
+export function countKeepAll(opts?: Oazapfts.RequestOpts) {
     return oazapfts.ok(oazapfts.fetchJson<{
         status: 200;
-        data: DuplicateResponseDto[];
-    }>("/duplicates", {
+        data: number;
+    }>("/duplicates/keep-all/count", {
         ...opts
     }));
 }

pnpm-lock.yaml

@@ -469,6 +469,9 @@ importers:
       lodash:
         specifier: ^4.17.21
         version: 4.17.21
+      lodash-es:
+        specifier: ^4.17.21
+        version: 4.17.21
       luxon:
         specifier: ^3.4.2
         version: 3.7.2
@@ -605,6 +608,9 @@ importers:
       '@types/lodash':
         specifier: ^4.14.197
         version: 4.17.21
+      '@types/lodash-es':
+        specifier: ^4.17.12
+        version: 4.17.12
       '@types/luxon':
         specifier: ^3.6.2
         version: 3.7.1

server/package.json

@@ -84,6 +84,7 @@
     "kysely": "0.28.2",
     "kysely-postgres-js": "^3.0.0",
     "lodash": "^4.17.21",
+    "lodash-es": "^4.17.21",
     "luxon": "^3.4.2",
     "mnemonist": "^0.40.3",
     "multer": "^2.0.2",
@@ -131,6 +132,7 @@
     "@types/jsonwebtoken": "^9.0.10",
     "@types/js-yaml": "^4.0.9",
     "@types/lodash": "^4.14.197",
+    "@types/lodash-es": "^4.17.12",
     "@types/luxon": "^3.6.2",
     "@types/mock-fs": "^4.13.1",
     "@types/multer": "^2.0.0",

server/src/controllers/duplicate.controller.ts

@@ -1,5 +1,5 @@
-import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param } from '@nestjs/common';
-import { ApiTags } from '@nestjs/swagger';
+import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, Query } from '@nestjs/common';
+import { ApiQuery, ApiTags } from '@nestjs/swagger';
 import { Endpoint, HistoryBuilder } from 'src/decorators';
 import { BulkIdsDto } from 'src/dtos/asset-ids.response.dto';
 import { AuthDto } from 'src/dtos/auth.dto';
@@ -15,14 +15,46 @@ export class DuplicateController {
   constructor(private service: DuplicateService) {}
 
   @Get()
+  @ApiQuery({ name: 'page', required: false, type: Number, example: 1 })
+  @ApiQuery({ name: 'size', required: false, type: Number, example: 20 })
   @Authenticated({ permission: Permission.DuplicateRead })
   @Endpoint({
     summary: 'Retrieve duplicates',
     description: 'Retrieve a list of duplicate assets available to the authenticated user.',
     history: new HistoryBuilder().added('v1').beta('v1').stable('v2'),
   })
-  getAssetDuplicates(@Auth() auth: AuthDto): Promise<DuplicateResponseDto[]> {
-    return this.service.getDuplicates(auth);
+  getAssetDuplicates(
+    @Auth() auth: AuthDto,
+    @Query('page') page: number = 1,
+    @Query('size') size: number = 20,
+  ): Promise<DuplicateResponseDto> {
+    return this.service.getDuplicates(auth, page, size);
+  }
+
+  @Get('/de-duplicate-all/count')
+  @Authenticated({ permission: Permission.DuplicateRead })
+  countDeDuplicateAll(@Auth() auth: AuthDto): Promise<number> {
+    return this.service.countDeDuplicateAll(auth);
+  }
+
+  @Delete('/de-duplicate-all')
+  @Authenticated({ permission: Permission.DuplicateDelete })
+  @HttpCode(HttpStatus.NO_CONTENT)
+  deDuplicateAll(@Auth() auth: AuthDto) {
+    return this.service.deDuplicateAll(auth);
+  }
+
+  @Get('/keep-all/count')
+  @Authenticated({ permission: Permission.DuplicateRead })
+  countKeepAll(@Auth() auth: AuthDto): Promise<number> {
+    return this.service.countKeepAll(auth);
+  }
+
+  @Delete('/keep-all')
+  @Authenticated({ permission: Permission.DuplicateDelete })
+  @HttpCode(HttpStatus.NO_CONTENT)
+  keepAll(@Auth() auth: AuthDto) {
+    return this.service.keepAll(auth);
   }
 
   @Delete()

server/src/dtos/duplicate.dto.ts

@@ -1,6 +1,14 @@
 import { AssetResponseDto } from 'src/dtos/asset-response.dto';
+import { PaginationResult } from 'src/utils/pagination';
 
-export class DuplicateResponseDto {
+class DuplicateItem {
   duplicateId!: string;
   assets!: AssetResponseDto[];
 }
+
+export class DuplicateResponseDto implements PaginationResult<DuplicateItem> {
+  items!: DuplicateItem[];
+  hasNextPage!: boolean;
+  totalPages!: number;
+  totalItems!: number;
+}

server/src/queries/duplicate.repository.sql

@@ -59,8 +59,70 @@ where
     where
       "unique"."duplicateId" = "duplicates"."duplicateId"
   )
+limit
+  $4
+offset
+  $5
 
 -- DuplicateRepository.delete
+with
+  "duplicates" as (
+    select
+      "asset"."duplicateId",
+      json_agg(
+        "asset2"
+        order by
+          "asset"."localDateTime" asc
+      ) as "assets"
+    from
+      "asset"
+      inner join lateral (
+        select
+          "asset".*,
+          "asset_exif" as "exifInfo"
+        from
+          "asset_exif"
+        where
+          "asset_exif"."assetId" = "asset"."id"
+      ) as "asset2" on true
+    where
+      "asset"."visibility" in ('archive', 'timeline')
+      and "asset"."ownerId" = $1::uuid
+      and "asset"."duplicateId" is not null
+      and "asset"."deletedAt" is null
+      and "asset"."stackId" is null
+    group by
+      "asset"."duplicateId"
+  ),
+  "unique" as (
+    select
+      "duplicateId"
+    from
+      "duplicates"
+    where
+      json_array_length("assets") = $2
+  ),
+  "removed_unique" as (
+    update "asset"
+    set
+      "duplicateId" = $3
+    from
+      "unique"
+    where
+      "asset"."duplicateId" = "unique"."duplicateId"
+  )
+select
+  count(*) as "count"
+from
+  "duplicates"
+where
+  not exists (
+    select
+    from
+      "unique"
+    where
+      "unique"."duplicateId" = "duplicates"."duplicateId"
+  )
 update "asset"
 set
   "duplicateId" = $1

server/src/repositories/duplicate.repository.ts

@@ -27,55 +27,69 @@ export class DuplicateRepository {
   constructor(@InjectKysely() private db: Kysely<DB>) {}
 
   @GenerateSql({ params: [DummyValue.UUID] })
-  getAll(userId: string) {
-    return (
-      this.db
-        .with('duplicates', (qb) =>
-          qb
-            .selectFrom('asset')
-            .$call(withDefaultVisibility)
-            .innerJoinLateral(
-              (qb) =>
-                qb
-                  .selectFrom('asset_exif')
-                  .selectAll('asset')
-                  .select((eb) => eb.table('asset_exif').as('exifInfo'))
-                  .whereRef('asset_exif.assetId', '=', 'asset.id')
-                  .as('asset2'),
-              (join) => join.onTrue(),
-            )
-            .select('asset.duplicateId')
-            .select((eb) =>
-              eb.fn.jsonAgg('asset2').orderBy('asset.localDateTime', 'asc').$castTo<MapAsset[]>().as('assets'),
-            )
-            .where('asset.ownerId', '=', asUuid(userId))
-            .where('asset.duplicateId', 'is not', null)
-            .$narrowType<{ duplicateId: NotNull }>()
-            .where('asset.deletedAt', 'is', null)
-            .where('asset.stackId', 'is', null)
-            .groupBy('asset.duplicateId'),
-        )
-        .with('unique', (qb) =>
-          qb
-            .selectFrom('duplicates')
-            .select('duplicateId')
-            .where((eb) => eb(eb.fn('json_array_length', ['assets']), '=', 1)),
-        )
-        .with('removed_unique', (qb) =>
-          qb
-            .updateTable('asset')
-            .set({ duplicateId: null })
-            .from('unique')
-            .whereRef('asset.duplicateId', '=', 'unique.duplicateId'),
-        )
-        .selectFrom('duplicates')
-        .selectAll()
-        // TODO: compare with filtering by json_array_length > 1
-        .where(({ not, exists }) =>
-          not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))),
-        )
-        .execute()
-    );
+  async getAll(userId: string, page: number, size: number) {
+    const query = this.db
+      .with('duplicates', (qb) =>
+        qb
+          .selectFrom('asset')
+          .$call(withDefaultVisibility)
+          .innerJoinLateral(
+            (qb) =>
+              qb
+                .selectFrom('asset_exif')
+                .selectAll('asset')
+                .select((eb) => eb.table('asset_exif').as('exifInfo'))
+                .whereRef('asset_exif.assetId', '=', 'asset.id')
+                .as('asset2'),
+            (join) => join.onTrue(),
+          )
+          .select('asset.duplicateId')
+          .select((eb) =>
+            eb.fn.jsonAgg('asset2').orderBy('asset.localDateTime', 'asc').$castTo<MapAsset[]>().as('assets'),
+          )
+          .where('asset.ownerId', '=', asUuid(userId))
+          .where('asset.duplicateId', 'is not', null)
+          .$narrowType<{ duplicateId: NotNull }>()
+          .where('asset.deletedAt', 'is', null)
+          .where('asset.stackId', 'is', null)
+          .groupBy('asset.duplicateId'),
+      )
+      .with('unique', (qb) =>
+        qb
+          .selectFrom('duplicates')
+          .select('duplicateId')
+          .where((eb) => eb(eb.fn('json_array_length', ['assets']), '=', 1)),
+      )
+      .with('removed_unique', (qb) =>
+        qb
+          .updateTable('asset')
+          .set({ duplicateId: null })
+          .from('unique')
+          .whereRef('asset.duplicateId', '=', 'unique.duplicateId'),
+      )
+      .selectFrom('duplicates')
+      .selectAll()
+      // TODO: compare with filtering by json_array_length > 1
+      .where(({ not, exists }) =>
+        not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))),
+      );
+
+    const [items, totalItems] = await Promise.all([
+      query
+        .offset((page - 1) * size)
+        .limit(size)
+        .execute(),
+      query
+        .clearSelect()
+        .select((eb) => eb.fn.countAll<number>().as('count'))
+        .executeTakeFirstOrThrow()
+        .then((r) => r.count),
+    ]);
+
+    return {
+      items,
+      totalItems,
+    };
   }
 
   @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })

server/src/repositories/event.repository.ts

@@ -6,6 +6,7 @@ import { Socket } from 'socket.io';
 import { SystemConfig } from 'src/config';
 import { Asset } from 'src/database';
 import { EventConfig } from 'src/decorators';
+import { AssetBulkDeleteDto } from 'src/dtos/asset.dto';
 import { AuthDto } from 'src/dtos/auth.dto';
 import { ImmichWorker, JobStatus, MetadataKey, QueueName, UserAvatarColor, UserStatus } from 'src/enum';
 import { ConfigRepository } from 'src/repositories/config.repository';
@@ -53,6 +54,7 @@ type EventMap = {
   AssetMetadataExtracted: [{ assetId: string; userId: string; source?: JobSource }];
 
   // asset bulk events
+  AssetDeleteRequest: [{ auth: AuthDto; dto: AssetBulkDeleteDto }];
   AssetTrashAll: [{ assetIds: string[]; userId: string }];
   AssetDeleteAll: [{ assetIds: string[]; userId: string }];
   AssetRestoreAll: [{ assetIds: string[]; userId: string }];

server/src/services/asset.service.ts

@@ -3,7 +3,7 @@ import _ from 'lodash';
 import { DateTime, Duration } from 'luxon';
 import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
 import { AssetFile } from 'src/database';
-import { OnJob } from 'src/decorators';
+import { OnEvent, OnJob } from 'src/decorators';
 import { AssetResponseDto, MapAsset, SanitizedAssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
 import {
   AssetBulkDeleteDto,
@@ -372,6 +372,7 @@ export class AssetService extends BaseService {
     return JobStatus.Success;
   }
 
+  @OnEvent({ name: 'AssetDeleteRequest' })
   async deleteAll(auth: AuthDto, dto: AssetBulkDeleteDto): Promise<void> {
     const { ids, force } = dto;
 

server/src/services/duplicate.service.spec.ts

@@ -38,21 +38,30 @@ describe(SearchService.name, () => {
 
   describe('getDuplicates', () => {
     it('should get duplicates', async () => {
-      mocks.duplicateRepository.getAll.mockResolvedValue([
-        {
-          duplicateId: 'duplicate-id',
-          assets: [assetStub.image, assetStub.image],
-        },
-      ]);
-      await expect(sut.getDuplicates(authStub.admin)).resolves.toEqual([
-        {
-          duplicateId: 'duplicate-id',
-          assets: [
-            expect.objectContaining({ id: assetStub.image.id }),
-            expect.objectContaining({ id: assetStub.image.id }),
-          ],
-        },
-      ]);
+      mocks.duplicateRepository.getAll.mockResolvedValue({
+        items: [
+          {
+            duplicateId: 'duplicate-id',
+            assets: [assetStub.image, assetStub.image],
+          },
+        ],
+        totalItems: 1,
+      });
+
+      await expect(sut.getDuplicates(authStub.admin)).resolves.toEqual({
+        items: [
+          {
+            duplicateId: 'duplicate-id',
+            assets: [
+              expect.objectContaining({ id: assetStub.image.id }),
+              expect.objectContaining({ id: assetStub.image.id }),
+            ],
+          },
+        ],
+        totalItems: expect.any(Number),
+        totalPages: expect.any(Number),
+        hasNextPage: expect.any(Boolean),
+      });
     });
   });
 

server/src/services/duplicate.service.ts

@@ -1,4 +1,4 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable } from '@nestjs/common';
 import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
 import { OnJob } from 'src/decorators';
 import { BulkIdsDto } from 'src/dtos/asset-ids.response.dto';
@@ -7,18 +7,33 @@ import { AuthDto } from 'src/dtos/auth.dto';
 import { DuplicateResponseDto } from 'src/dtos/duplicate.dto';
 import { AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum';
 import { AssetDuplicateResult } from 'src/repositories/search.repository';
+import { AssetService } from 'src/services/asset.service';
 import { BaseService } from 'src/services/base.service';
 import { JobItem, JobOf } from 'src/types';
+import { suggestDuplicate } from 'src/utils/duplicate-utils';
 import { isDuplicateDetectionEnabled } from 'src/utils/misc';
 
 @Injectable()
 export class DuplicateService extends BaseService {
-  async getDuplicates(auth: AuthDto): Promise<DuplicateResponseDto[]> {
-    const duplicates = await this.duplicateRepository.getAll(auth.user.id);
-    return duplicates.map(({ duplicateId, assets }) => ({
+  @Inject() private assetService!: AssetService;
+
+  async getDuplicates(auth: AuthDto, page = 1, size = 20): Promise<DuplicateResponseDto> {
+    const { items, totalItems } = await this.duplicateRepository.getAll(auth.user.id, page, size);
+
+    const duplicates = items.map(({ duplicateId, assets }) => ({
       duplicateId,
       assets: assets.map((asset) => mapAsset(asset, { auth })),
     }));
+
+    const totalPages = Math.ceil(totalItems / size);
+    const hasNextPage = page < totalPages;
+
+    return {
+      items: duplicates,
+      totalItems,
+      totalPages,
+      hasNextPage,
+    };
   }
 
   async delete(auth: AuthDto, id: string): Promise<void> {
@@ -29,6 +44,89 @@ export class DuplicateService extends BaseService {
     await this.duplicateRepository.deleteAll(auth.user.id, dto.ids);
   }
 
+  async countDeDuplicateAll(auth: AuthDto): Promise<number> {
+    let page = 1;
+    const size = 100;
+    let hasNextPage = true;
+    let totalToDelete = 0;
+
+    while (hasNextPage) {
+      const duplicates = await this.getDuplicates(auth, page, size);
+
+      const idsToKeep = duplicates.items.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id);
+      const idsToDelete = duplicates.items.flatMap((group, i) =>
+        group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]),
+      );
+
+      totalToDelete += idsToDelete.length;
+
+      hasNextPage = duplicates.hasNextPage;
+      page++;
+    }
+
+    return totalToDelete;
+  }
+
+  async deDuplicateAll(auth: AuthDto) {
+    let page = 1;
+    const size = 100;
+    let hasNextPage = true;
+
+    while (hasNextPage) {
+      const duplicates = await this.getDuplicates(auth, page, size);
+
+      const idsToKeep = duplicates.items.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id);
+      const idsToDelete = duplicates.items.flatMap((group, i) =>
+        group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]),
+      );
+
+      const { trash } = await this.getConfig({ withCache: false });
+
+      await this.eventRepository.emit('AssetDeleteRequest', {
+        auth,
+        dto: { ids: idsToDelete, force: !trash },
+      });
+
+      hasNextPage = duplicates.hasNextPage;
+      page++;
+    }
+  }
+
+  async countKeepAll(auth: AuthDto): Promise<number> {
+    let page = 1;
+    const size = 100;
+    let hasNextPage = true;
+    let totalToDelete = 0;
+
+    while (hasNextPage) {
+      const duplicates = await this.getDuplicates(auth, page, size);
+
+      totalToDelete += duplicates.items.length;
+
+      hasNextPage = duplicates.hasNextPage;
+      page++;
+    }
+
+    return totalToDelete;
+  }
+
+  async keepAll(auth: AuthDto) {
+    let page = 1;
+    const size = 100;
+    let hasNextPage = true;
+
+    while (hasNextPage) {
+      const duplicates = await this.getDuplicates(auth, page, size);
+
+      const idsToDelete = duplicates.items.map(({ duplicateId }) => duplicateId);
+
+      await this.deleteAll(auth, { ids: idsToDelete });
+
+      hasNextPage = duplicates.hasNextPage;
+      page++;
+    }
+  }
+
   @OnJob({ name: JobName.AssetDetectDuplicatesQueueAll, queue: QueueName.DuplicateDetection })
   async handleQueueSearchDuplicates({ force }: JobOf<JobName.AssetDetectDuplicatesQueueAll>): Promise<JobStatus> {
     const { machineLearning } = await this.getConfig({ withCache: false });

server/src/utils/duplicate-utils.spec.ts

@@ -0,0 +1,37 @@
+import { AssetResponseDto } from 'src/dtos/asset-response.dto';
+import { suggestDuplicate } from 'src/utils/duplicate-utils';
+
+describe('choosing a duplicate', () => {
+  it('picks the asset with the largest file size', () => {
+    const assets = [
+      { exifInfo: { fileSizeInByte: 300 } },
+      { exifInfo: { fileSizeInByte: 200 } },
+      { exifInfo: { fileSizeInByte: 100 } },
+    ];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+
+  it('picks the asset with the most exif data if multiple assets have the same file size', () => {
+    const assets = [
+      { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1 } },
+      { exifInfo: { fileSizeInByte: 200, rating: 5 } },
+      { exifInfo: { fileSizeInByte: 100, rating: 5 } },
+    ];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+
+  it('returns undefined for an empty array', () => {
+    const assets: AssetResponseDto[] = [];
+    expect(suggestDuplicate(assets)).toBeUndefined();
+  });
+
+  it('handles assets with no exifInfo', () => {
+    const assets = [{ exifInfo: { fileSizeInByte: 200 } }, {}];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+
+  it('handles assets with exifInfo but no fileSizeInByte', () => {
+    const assets = [{ exifInfo: { rating: 5, fNumber: 1 } }, { exifInfo: { rating: 5 } }];
+    expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]);
+  });
+});

server/src/utils/duplicate-utils.ts

@@ -0,0 +1,30 @@
+import { sortBy } from 'lodash';
+import { AssetResponseDto } from 'src/dtos/asset-response.dto';
+import { getExifCount } from 'src/utils/exif-utils';
+
+/**
+ * Suggests the best duplicate asset to keep from a list of duplicates.
+ *
+ * The best asset is determined by the following criteria:
+ *  - Largest image file size in bytes
+ *  - Largest count of exif data
+ *
+ * @param assets List of duplicate assets
+ * @returns The best asset to keepweb/src/lib/utils/duplicate-utils.spec.ts
+ */
+export const suggestDuplicate = (assets: AssetResponseDto[]): AssetResponseDto | undefined => {
+  let duplicateAssets = sortBy(assets, (asset) => asset.exifInfo?.fileSizeInByte ?? 0);
+
+  // Update the list to only include assets with the largest file size
+  duplicateAssets = duplicateAssets.filter(
+    (asset) => asset.exifInfo?.fileSizeInByte === duplicateAssets.at(-1)?.exifInfo?.fileSizeInByte,
+  );
+
+  // If there are multiple assets with the same file size, sort the list by the count of exif data
+  if (duplicateAssets.length >= 2) {
+    duplicateAssets = sortBy(duplicateAssets, getExifCount);
+  }
+
+  // Return the last asset in the list
+  return duplicateAssets.pop();
+};

server/src/utils/exif-utils.spec.ts

@@ -0,0 +1,29 @@
+import { AssetResponseDto } from 'src/dtos/asset-response.dto';
+import { getExifCount } from 'src/utils/exif-utils';
+
+describe('getting the exif count', () => {
+  it('returns 0 when exifInfo is undefined', () => {
+    const asset = {};
+    expect(getExifCount(asset as AssetResponseDto)).toBe(0);
+  });
+
+  it('returns 0 when exifInfo is empty', () => {
+    const asset = { exifInfo: {} };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(0);
+  });
+
+  it('returns the correct count of non-null exifInfo properties', () => {
+    const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: null } };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(2);
+  });
+
+  it('ignores null, undefined and empty properties in exifInfo', () => {
+    const asset = { exifInfo: { fileSizeInByte: 200, rating: null, fNumber: undefined, description: '' } };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(1);
+  });
+
+  it('returns the correct count when all exifInfo properties are non-null', () => {
+    const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1, description: 'test' } };
+    expect(getExifCount(asset as AssetResponseDto)).toBe(4);
+  });
+});

server/src/utils/exif-utils.ts

@@ -0,0 +1,5 @@
+import { AssetResponseDto } from 'src/dtos/asset-response.dto';
+
+export const getExifCount = (asset: AssetResponseDto) => {
+  return Object.values(asset.exifInfo ?? {}).filter(Boolean).length;
+};

web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte

@@ -11,10 +11,17 @@
   import { assetViewingStore } from '$lib/stores/asset-viewing.store';
   import { locale } from '$lib/stores/preferences.store';
   import { stackAssets } from '$lib/utils/asset-utils';
-  import { suggestDuplicate } from '$lib/utils/duplicate-utils';
   import { handleError } from '$lib/utils/handle-error';
   import type { AssetResponseDto } from '@immich/sdk';
-  import { deleteAssets, deleteDuplicates, updateAssets } from '@immich/sdk';
+  import {
+    countDeDuplicateAll,
+    countKeepAll,
+    deDuplicateAll,
+    deleteAssets,
+    getAssetDuplicates,
+    keepAll,
+    updateAssets,
+  } from '@immich/sdk';
   import { Button, HStack, IconButton, modalManager, Text, toastManager } from '@immich/ui';
   import {
     mdiCheckOutline,
@@ -35,6 +42,8 @@
 
   let { data = $bindable() }: Props = $props();
 
+  const PAGE_SIZE = data.pageSize;
+
   interface Shortcuts {
     general: ExplainedShortcut[];
     actions: ExplainedShortcut[];
@@ -56,11 +65,20 @@
     ],
   };
 
-  let duplicates = $state(data.duplicates);
+  let duplicatesRes = $state(data.duplicatesRes);
+  let pageCache = $state<Map<number, typeof duplicatesRes>>(new Map());
+
+  $effect(() => {
+    const initialPage = Math.floor(duplicatesIndex / PAGE_SIZE) + 1;
+    if (!pageCache.has(initialPage)) {
+      pageCache.set(initialPage, duplicatesRes);
+    }
+  });
+
   const { isViewing: showAssetViewer } = assetViewingStore;
 
   const correctDuplicatesIndex = (index: number) => {
-    return Math.max(0, Math.min(index, duplicates.length - 1));
+    return Math.max(0, Math.min(index, duplicatesRes.totalItems - 1));
   };
 
   let duplicatesIndex = $derived(
@@ -71,7 +89,7 @@
     })(),
   );
 
-  let hasDuplicates = $derived(duplicates.length > 0);
+  let hasDuplicates = $derived(duplicatesRes.totalItems > 0);
   const withConfirmation = async (callback: () => Promise<void>, prompt?: string, confirmText?: string) => {
     if (prompt && confirmText) {
       const isConfirmed = await modalManager.showDialog({ prompt, confirmText });
@@ -98,14 +116,12 @@
     toastManager.success(message);
   };
 
-  const handleResolve = async (duplicateId: string, duplicateAssetIds: string[], trashIds: string[]) => {
+  const handleResolve = async (duplicateAssetIds: string[], trashIds: string[]) => {
     return withConfirmation(
       async () => {
         await deleteAssets({ assetBulkDeleteDto: { ids: trashIds, force: !featureFlagsManager.value.trash } });
         await updateAssets({ assetBulkUpdateDto: { ids: duplicateAssetIds, duplicateId: null } });
 
-        duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId);
-
         deletedNotification(trashIds.length);
         await correctDuplicatesIndexAndGo(duplicatesIndex);
       },
@@ -114,42 +130,30 @@
     );
   };
 
-  const handleStack = async (duplicateId: string, assets: AssetResponseDto[]) => {
+  const handleStack = async (assets: AssetResponseDto[]) => {
     await stackAssets(assets, false);
     const duplicateAssetIds = assets.map((asset) => asset.id);
     await updateAssets({ assetBulkUpdateDto: { ids: duplicateAssetIds, duplicateId: null } });
-    duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId);
     await correctDuplicatesIndexAndGo(duplicatesIndex);
   };
 
   const handleDeduplicateAll = async () => {
-    const idsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id);
-    const idsToDelete = duplicates.flatMap((group, i) =>
-      group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]),
-    );
-
+    const count = await countDeDuplicateAll();
     let prompt, confirmText;
     if (featureFlagsManager.value.trash) {
-      prompt = $t('bulk_trash_duplicates_confirmation', { values: { count: idsToDelete.length } });
+      prompt = $t('bulk_trash_duplicates_confirmation', { values: { count } });
       confirmText = $t('confirm');
     } else {
-      prompt = $t('bulk_delete_duplicates_confirmation', { values: { count: idsToDelete.length } });
+      prompt = $t('bulk_delete_duplicates_confirmation', { values: { count } });
       confirmText = $t('permanently_delete');
     }
 
     return withConfirmation(
       async () => {
-        await deleteAssets({ assetBulkDeleteDto: { ids: idsToDelete, force: !featureFlagsManager.value.trash } });
-        await updateAssets({
-          assetBulkUpdateDto: {
-            ids: [...idsToDelete, ...idsToKeep.filter((id): id is string => !!id)],
-            duplicateId: null,
-          },
-        });
+        await deDuplicateAll();
+        deletedNotification(1);
 
-        duplicates = [];
-
-        deletedNotification(idsToDelete.length);
+        duplicatesRes.items = [];
 
         page.url.searchParams.delete('index');
         await goto(`${AppRoute.DUPLICATES}`);
@@ -160,18 +164,16 @@
   };
 
   const handleKeepAll = async () => {
-    const ids = duplicates.map(({ duplicateId }) => duplicateId);
+    const count = await countKeepAll();
     return withConfirmation(
       async () => {
-        await deleteDuplicates({ bulkIdsDto: { ids } });
-
-        duplicates = [];
+        await keepAll();
 
         toastManager.success($t('resolved_all_duplicates'));
         page.url.searchParams.delete('index');
         await goto(`${AppRoute.DUPLICATES}`);
       },
-      $t('bulk_keep_duplicates_confirmation', { values: { count: ids.length } }),
+      $t('bulk_keep_duplicates_confirmation', { values: { count } }),
       $t('confirm'),
     );
   };
@@ -179,30 +181,79 @@
   const handleFirst = async () => {
     await correctDuplicatesIndexAndGo(0);
   };
+
   const handlePrevious = async () => {
     await correctDuplicatesIndexAndGo(Math.max(duplicatesIndex - 1, 0));
   };
+
   const handlePreviousShortcut = async () => {
     if ($showAssetViewer) {
       return;
     }
     await handlePrevious();
   };
+
   const handleNext = async () => {
-    await correctDuplicatesIndexAndGo(Math.min(duplicatesIndex + 1, duplicates.length - 1));
+    await correctDuplicatesIndexAndGo(Math.min(duplicatesIndex + 1, duplicatesRes.totalItems - 1));
   };
+
   const handleNextShortcut = async () => {
     if ($showAssetViewer) {
       return;
     }
     await handleNext();
   };
+
   const handleLast = async () => {
-    await correctDuplicatesIndexAndGo(duplicates.length - 1);
+    await correctDuplicatesIndexAndGo(duplicatesRes.totalItems - 1);
   };
+
   const correctDuplicatesIndexAndGo = async (index: number) => {
-    page.url.searchParams.set('index', correctDuplicatesIndex(index).toString());
+    const correctedIndex = correctDuplicatesIndex(index);
+    const pageNeeded = Math.floor(correctedIndex / PAGE_SIZE) + 1;
+    const currentPage = Math.floor(duplicatesIndex / PAGE_SIZE) + 1;
+
+    if (pageNeeded !== currentPage || !pageCache.has(pageNeeded)) {
+      await loadDuplicates(pageNeeded);
+    } else {
+      duplicatesRes = pageCache.get(pageNeeded)!;
+    }
+
+    page.url.searchParams.set('index', correctedIndex.toString());
     await goto(`${AppRoute.DUPLICATES}?${page.url.searchParams.toString()}`);
+
+    void preloadAdjacentPages(pageNeeded, correctedIndex);
+  };
+
+  const loadDuplicates = async (pageNumber: number) => {
+    if (pageCache.has(pageNumber)) {
+      duplicatesRes = pageCache.get(pageNumber)!;
+      return;
+    }
+
+    duplicatesRes = await getAssetDuplicates({ page: pageNumber, size: PAGE_SIZE });
+    pageCache.set(pageNumber, duplicatesRes);
+  };
+
+  const preloadAdjacentPages = async (currentPageNumber: number, currentIndex: number) => {
+    const localIndex = currentIndex % PAGE_SIZE;
+    const maxPage = Math.ceil(duplicatesRes.totalItems / PAGE_SIZE);
+
+    if (localIndex === PAGE_SIZE - 1 && currentPageNumber < maxPage) {
+      const nextPage = currentPageNumber + 1;
+      if (!pageCache.has(nextPage)) {
+        const res = await getAssetDuplicates({ page: nextPage, size: PAGE_SIZE });
+        pageCache.set(nextPage, res);
+      }
+    }
+
+    if (localIndex === 0 && currentPageNumber > 1) {
+      const prevPage = currentPageNumber - 1;
+      if (!pageCache.has(prevPage)) {
+        const res = await getAssetDuplicates({ page: prevPage, size: PAGE_SIZE });
+        pageCache.set(prevPage, res);
+      }
+    }
   };
 </script>
 
@@ -213,7 +264,7 @@
   ]}
 />
 
-<UserPageLayout title={data.meta.title + ` (${duplicates.length.toLocaleString($locale)})`} scrollbar={true}>
+<UserPageLayout title={data.meta.title + ` (${duplicatesRes.totalItems.toLocaleString($locale)})`} scrollbar={true}>
   {#snippet buttons()}
     <HStack gap={0}>
       <Button
@@ -248,8 +299,11 @@
     </HStack>
   {/snippet}
 
-  <div class="">
-    {#if duplicates && duplicates.length > 0}
+  <div>
+    {#if duplicatesRes.items.length > 0 && duplicatesRes.totalItems > 0}
+      {@const localIndex = duplicatesIndex % PAGE_SIZE}
+      {@const currentDuplicate = duplicatesRes.items[localIndex]}
+
       <div class="flex items-center mb-2">
         <div class="text-sm dark:text-white">
           <p>{$t('duplicates_description')}</p>
@@ -265,65 +319,64 @@
         />
       </div>
 
-      {#key duplicates[duplicatesIndex].duplicateId}
-        <DuplicatesCompareControl
-          assets={duplicates[duplicatesIndex].assets}
-          onResolve={(duplicateAssetIds, trashIds) =>
-            handleResolve(duplicates[duplicatesIndex].duplicateId, duplicateAssetIds, trashIds)}
-          onStack={(assets) => handleStack(duplicates[duplicatesIndex].duplicateId, assets)}
-        />
-        <div class="max-w-5xl mx-auto mb-16">
-          <div class="flex mb-4 sm:px-6 w-full place-content-center justify-between items-center place-items-center">
-            <div class="flex text-xs text-black">
-              <Button
-                size="small"
-                leadingIcon={mdiPageFirst}
-                color="primary"
-                class="flex place-items-center rounded-s-full gap-2 px-2 sm:px-4"
-                onclick={handleFirst}
-                disabled={duplicatesIndex === 0}
-              >
-                {$t('first')}
-              </Button>
-              <Button
-                size="small"
-                leadingIcon={mdiChevronLeft}
-                color="primary"
-                class="flex place-items-center rounded-e-full gap-2 px-2 sm:px-4"
-                onclick={handlePrevious}
-                disabled={duplicatesIndex === 0}
-              >
-                {$t('previous')}
-              </Button>
-            </div>
-            <p class="border px-3 md:px-6 py-1 dark:bg-subtle rounded-lg text-xs md:text-sm">
-              {duplicatesIndex + 1} / {duplicates.length.toLocaleString($locale)}
-            </p>
-            <div class="flex text-xs text-black">
-              <Button
-                size="small"
-                trailingIcon={mdiChevronRight}
-                color="primary"
-                class="flex place-items-center rounded-s-full gap-2 px-2 sm:px-4"
-                onclick={handleNext}
-                disabled={duplicatesIndex === duplicates.length - 1}
-              >
-                {$t('next')}
-              </Button>
-              <Button
-                size="small"
-                trailingIcon={mdiPageLast}
-                color="primary"
-                class="flex place-items-center rounded-e-full gap-2 px-2 sm:px-4"
-                onclick={handleLast}
-                disabled={duplicatesIndex === duplicates.length - 1}
-              >
-                {$t('last')}
-              </Button>
-            </div>
+      {#if currentDuplicate}
+        {#key currentDuplicate.duplicateId}
+          <DuplicatesCompareControl
+            assets={currentDuplicate.assets}
+            onResolve={(duplicateAssetIds, trashIds) => handleResolve(duplicateAssetIds, trashIds)}
+            onStack={(assets) => handleStack(assets)}
+          />
+        {/key}
+      {/if}
+      <div class="max-w-5xl mx-auto mb-16">
+        <div class="flex mb-4 sm:px-6 w-full place-content-center justify-between items-center place-items-center">
+          <div class="flex text-xs text-black">
+            <Button
+              size="small"
+              leadingIcon={mdiPageFirst}
+              color="primary"
+              class="flex place-items-center rounded-s-full gap-2 px-2 sm:px-4"
+              onclick={handleFirst}
+              disabled={duplicatesIndex === 0}
+            >
+              {$t('first')}
+            </Button>
+            <Button
+              size="small"
+              leadingIcon={mdiChevronLeft}
+              color="primary"
+              class="flex place-items-center rounded-e-full gap-2 px-2 sm:px-4"
+              onclick={handlePrevious}
+              disabled={duplicatesIndex === 0}
+            >
+              {$t('previous')}
+            </Button>
+          </div>
+          <p>{duplicatesIndex + 1}/{duplicatesRes.totalItems.toLocaleString($locale)}</p>
+          <div class="flex text-xs text-black">
+            <Button
+              size="small"
+              trailingIcon={mdiChevronRight}
+              color="primary"
+              class="flex place-items-center rounded-s-full gap-2 px-2 sm:px-4"
+              onclick={handleNext}
+              disabled={duplicatesIndex === duplicatesRes.totalItems - 1}
+            >
+              {$t('next')}
+            </Button>
+            <Button
+              size="small"
+              trailingIcon={mdiPageLast}
+              color="primary"
+              class="flex place-items-center rounded-e-full gap-2 px-2 sm:px-4"
+              onclick={handleLast}
+              disabled={duplicatesIndex === duplicatesRes.totalItems - 1}
+            >
+              {$t('last')}
+            </Button>
           </div>
         </div>
-      {/key}
+      </div>
     {:else}
       <p class="text-center text-lg dark:text-white flex place-items-center place-content-center">
         {$t('no_duplicates_found')}

web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.ts

@@ -7,12 +7,20 @@ import type { PageLoad } from './$types';
 export const load = (async ({ params, url }) => {
   await authenticate(url);
   const asset = await getAssetInfoFromParam(params);
-  const duplicates = await getAssetDuplicates();
   const $t = await getFormatter();
 
+  const PAGE_SIZE = 10;
+
+  const indexParam = url.searchParams.get('index') ?? '0';
+  const parsedIndex = Number.parseInt(indexParam, 10);
+
+  const pageNumber = Math.floor(parsedIndex / PAGE_SIZE) + 1;
+  const duplicatesRes = await getAssetDuplicates({ page: pageNumber, size: PAGE_SIZE });
+
   return {
     asset,
-    duplicates,
+    duplicatesRes,
+    pageSize: PAGE_SIZE,
     meta: {
       title: $t('duplicates'),
     },