From c9b4012abee3647acfdeb6519827179fb12002ab Mon Sep 17 00:00:00 2001 From: MontejoJorge Date: Tue, 21 Oct 2025 20:30:18 +0200 Subject: [PATCH] deDuplicateAll and keepAll endpoints --- open-api/immich-openapi-specs.json | 54 +++++++++++++ open-api/typescript-sdk/src/fetch-client.ts | 18 +++++ pnpm-lock.yaml | 8 +- server/package.json | 2 + .../src/controllers/duplicate.controller.ts | 16 +++- server/src/services/duplicate.service.ts | 58 ++++++++++++- server/src/utils/duplicate-utils.spec.ts | 38 +++++++++ server/src/utils/duplicate-utils.ts | 30 +++++++ server/src/utils/exif-utils.spec.ts | 30 +++++++ server/src/utils/exif-utils.ts | 6 ++ .../[[assetId=id]]/+page.svelte | 81 ++++++++----------- .../[[photos=photos]]/[[assetId=id]]/+page.ts | 16 +++- 12 files changed, 301 insertions(+), 56 deletions(-) create mode 100644 server/src/utils/duplicate-utils.spec.ts create mode 100644 server/src/utils/duplicate-utils.ts create mode 100644 server/src/utils/exif-utils.spec.ts create mode 100644 server/src/utils/exif-utils.ts diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 603ed33c4d..2000b46e09 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -4645,6 +4645,60 @@ "x-immich-state": "Stable" } }, + "/duplicates/de-duplicate-all": { + "delete": { + "operationId": "deDuplicateAll", + "parameters": [], + "responses": { + "204": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Duplicates" + ], + "x-immich-permission": "duplicate.delete", + "description": "This endpoint requires the `duplicate.delete` permission." + } + }, + "/duplicates/keep-all": { + "delete": { + "operationId": "keepAll", + "parameters": [], + "responses": { + "204": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Duplicates" + ], + "x-immich-permission": "duplicate.delete", + "description": "This endpoint requires the `duplicate.delete` permission." + } + }, "/duplicates/{id}": { "delete": { "description": "Delete a single duplicate asset specified by its ID.", diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 70ee694d32..20098f1040 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -2885,6 +2885,24 @@ export function getAssetDuplicates({ page, size }: { /** * Delete a duplicate */ +export function deDuplicateAll(opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/duplicates/de-duplicate-all", { + ...opts, + method: "DELETE" + })); +} +/** + * This endpoint requires the `duplicate.delete` permission. + */ +export function keepAll(opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/duplicates/keep-all", { + ...opts, + method: "DELETE" + })); +} +/** + * This endpoint requires the `duplicate.delete` permission. + */ export function deleteDuplicate({ id }: { id: string; }, opts?: Oazapfts.RequestOpts) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95e2d4945e..2903875c38 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -469,6 +469,9 @@ importers: lodash: specifier: ^4.17.21 version: 4.17.21 + lodash-es: + specifier: ^4.17.21 + version: 4.17.21 luxon: specifier: ^3.4.2 version: 3.7.2 @@ -604,7 +607,10 @@ importers: version: 9.0.10 '@types/lodash': specifier: ^4.14.197 - version: 4.17.21 + version: 4.17.20 + '@types/lodash-es': + specifier: ^4.17.12 + version: 4.17.12 '@types/luxon': specifier: ^3.6.2 version: 3.7.1 diff --git a/server/package.json b/server/package.json index 915e45c116..5b554c106e 100644 --- a/server/package.json +++ b/server/package.json @@ -84,6 +84,7 @@ "kysely": "0.28.2", "kysely-postgres-js": "^3.0.0", "lodash": "^4.17.21", + "lodash-es": "^4.17.21", "luxon": "^3.4.2", "mnemonist": "^0.40.3", "multer": "^2.0.2", @@ -131,6 +132,7 @@ "@types/jsonwebtoken": "^9.0.10", "@types/js-yaml": "^4.0.9", "@types/lodash": "^4.14.197", + "@types/lodash-es": "^4.17.12", "@types/luxon": "^3.6.2", "@types/mock-fs": "^4.13.1", "@types/multer": "^2.0.0", diff --git a/server/src/controllers/duplicate.controller.ts b/server/src/controllers/duplicate.controller.ts index 85625060b4..a10ef98453 100644 --- a/server/src/controllers/duplicate.controller.ts +++ b/server/src/controllers/duplicate.controller.ts @@ -1,4 +1,4 @@ -import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, Query } from '@nestjs/common'; +import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, Post, Query } from '@nestjs/common'; import { ApiQuery, ApiTags } from '@nestjs/swagger'; import { Endpoint, HistoryBuilder } from 'src/decorators'; import { BulkIdsDto } from 'src/dtos/asset-ids.response.dto'; @@ -28,6 +28,20 @@ export class DuplicateController { return this.service.getDuplicates(auth, page, size); } + @Delete('/de-duplicate-all') + @Authenticated({ permission: Permission.DuplicateDelete }) + @HttpCode(HttpStatus.NO_CONTENT) + deDuplicateAll(@Auth() auth: AuthDto) { + return this.service.deDuplicateAll(auth); + } + + @Delete('/keep-all') + @Authenticated({ permission: Permission.DuplicateDelete }) + @HttpCode(HttpStatus.NO_CONTENT) + keepAll(@Auth() auth: AuthDto) { + return this.service.keepAll(auth); + } + @Delete() @Authenticated({ permission: Permission.DuplicateDelete }) @HttpCode(HttpStatus.NO_CONTENT) diff --git a/server/src/services/duplicate.service.ts b/server/src/services/duplicate.service.ts index 0c61967979..54983b8846 100644 --- a/server/src/services/duplicate.service.ts +++ b/server/src/services/duplicate.service.ts @@ -1,18 +1,22 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants'; import { OnJob } from 'src/decorators'; import { BulkIdsDto } from 'src/dtos/asset-ids.response.dto'; import { mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { DuplicateResponseDto } from 'src/dtos/duplicate.dto'; -import { AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum'; +import { AssetStatus, AssetVisibility, JobName, JobStatus, Permission, QueueName } from 'src/enum'; import { AssetDuplicateResult } from 'src/repositories/search.repository'; +import { AssetService } from 'src/services/asset.service'; import { BaseService } from 'src/services/base.service'; import { JobItem, JobOf } from 'src/types'; +import { suggestDuplicate } from 'src/utils/duplicate-utils'; import { isDuplicateDetectionEnabled } from 'src/utils/misc'; @Injectable() export class DuplicateService extends BaseService { + @Inject() private assetService!: AssetService; + async getDuplicates(auth: AuthDto, page = 1, size = 20): Promise { const { items, totalItems } = await this.duplicateRepository.getAll(auth.user.id, page, size); @@ -23,12 +27,12 @@ export class DuplicateService extends BaseService { const totalPages = Math.ceil(totalItems / size); const hasNextPage = page < totalPages; - + return { items: duplicates, totalItems, totalPages, - hasNextPage + hasNextPage, }; } @@ -40,6 +44,52 @@ export class DuplicateService extends BaseService { await this.duplicateRepository.deleteAll(auth.user.id, dto.ids); } + async deDuplicateAll(auth: AuthDto) { + let page = 1; + const size = 100; + let hasNextPage = true; + + while (hasNextPage) { + const duplicates = await this.getDuplicates(auth, page, size); + + const idsToKeep = duplicates.items.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id); + const idsToDelete = duplicates.items.flatMap((group, i) => + group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]), + ); + + // This is duplicated from asset.service - deleteAll() + await this.requireAccess({ auth, permission: Permission.AssetDelete, ids: idsToDelete }); + await this.assetRepository.updateAll(idsToDelete, { + deletedAt: new Date(), + status: false ? AssetStatus.Deleted : AssetStatus.Trashed, + }); + await this.eventRepository.emit(false ? 'AssetDeleteAll' : 'AssetTrashAll', { + assetIds: idsToDelete, + userId: auth.user.id, + }); + + hasNextPage = duplicates.hasNextPage; + page++; + } + } + + async keepAll(auth: AuthDto) { + let page = 1; + const size = 100; + let hasNextPage = true; + + while (hasNextPage) { + const duplicates = await this.getDuplicates(auth, page, size); + + const idsToDelete = duplicates.items.map(({ duplicateId }) => duplicateId); + + await this.deleteAll(auth, { ids: idsToDelete }); + + hasNextPage = duplicates.hasNextPage; + page++; + } + } + @OnJob({ name: JobName.AssetDetectDuplicatesQueueAll, queue: QueueName.DuplicateDetection }) async handleQueueSearchDuplicates({ force }: JobOf): Promise { const { machineLearning } = await this.getConfig({ withCache: false }); diff --git a/server/src/utils/duplicate-utils.spec.ts b/server/src/utils/duplicate-utils.spec.ts new file mode 100644 index 0000000000..313cf24dd3 --- /dev/null +++ b/server/src/utils/duplicate-utils.spec.ts @@ -0,0 +1,38 @@ +import { AssetResponseDto } from "src/dtos/asset-response.dto"; +import { suggestDuplicate } from "src/utils/duplicate-utils"; + + +describe('choosing a duplicate', () => { + it('picks the asset with the largest file size', () => { + const assets = [ + { exifInfo: { fileSizeInByte: 300 } }, + { exifInfo: { fileSizeInByte: 200 } }, + { exifInfo: { fileSizeInByte: 100 } }, + ]; + expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]); + }); + + it('picks the asset with the most exif data if multiple assets have the same file size', () => { + const assets = [ + { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1 } }, + { exifInfo: { fileSizeInByte: 200, rating: 5 } }, + { exifInfo: { fileSizeInByte: 100, rating: 5 } }, + ]; + expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]); + }); + + it('returns undefined for an empty array', () => { + const assets: AssetResponseDto[] = []; + expect(suggestDuplicate(assets)).toBeUndefined(); + }); + + it('handles assets with no exifInfo', () => { + const assets = [{ exifInfo: { fileSizeInByte: 200 } }, {}]; + expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]); + }); + + it('handles assets with exifInfo but no fileSizeInByte', () => { + const assets = [{ exifInfo: { rating: 5, fNumber: 1 } }, { exifInfo: { rating: 5 } }]; + expect(suggestDuplicate(assets as AssetResponseDto[])).toEqual(assets[0]); + }); +}); diff --git a/server/src/utils/duplicate-utils.ts b/server/src/utils/duplicate-utils.ts new file mode 100644 index 0000000000..d5508819c7 --- /dev/null +++ b/server/src/utils/duplicate-utils.ts @@ -0,0 +1,30 @@ +import { sortBy } from "lodash"; +import { AssetResponseDto } from "src/dtos/asset-response.dto"; +import { getExifCount } from "src/utils/exif-utils"; + +/** + * Suggests the best duplicate asset to keep from a list of duplicates. + * + * The best asset is determined by the following criteria: + * - Largest image file size in bytes + * - Largest count of exif data + * + * @param assets List of duplicate assets + * @returns The best asset to keepweb/src/lib/utils/duplicate-utils.spec.ts + */ +export const suggestDuplicate = (assets: AssetResponseDto[]): AssetResponseDto | undefined => { + let duplicateAssets = sortBy(assets, (asset) => asset.exifInfo?.fileSizeInByte ?? 0); + + // Update the list to only include assets with the largest file size + duplicateAssets = duplicateAssets.filter( + (asset) => asset.exifInfo?.fileSizeInByte === duplicateAssets.at(-1)?.exifInfo?.fileSizeInByte, + ); + + // If there are multiple assets with the same file size, sort the list by the count of exif data + if (duplicateAssets.length >= 2) { + duplicateAssets = sortBy(duplicateAssets, getExifCount); + } + + // Return the last asset in the list + return duplicateAssets.pop(); +}; diff --git a/server/src/utils/exif-utils.spec.ts b/server/src/utils/exif-utils.spec.ts new file mode 100644 index 0000000000..23811d1898 --- /dev/null +++ b/server/src/utils/exif-utils.spec.ts @@ -0,0 +1,30 @@ +import { AssetResponseDto } from "src/dtos/asset-response.dto"; +import { getExifCount } from "src/utils/exif-utils"; + + +describe('getting the exif count', () => { + it('returns 0 when exifInfo is undefined', () => { + const asset = {}; + expect(getExifCount(asset as AssetResponseDto)).toBe(0); + }); + + it('returns 0 when exifInfo is empty', () => { + const asset = { exifInfo: {} }; + expect(getExifCount(asset as AssetResponseDto)).toBe(0); + }); + + it('returns the correct count of non-null exifInfo properties', () => { + const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: null } }; + expect(getExifCount(asset as AssetResponseDto)).toBe(2); + }); + + it('ignores null, undefined and empty properties in exifInfo', () => { + const asset = { exifInfo: { fileSizeInByte: 200, rating: null, fNumber: undefined, description: '' } }; + expect(getExifCount(asset as AssetResponseDto)).toBe(1); + }); + + it('returns the correct count when all exifInfo properties are non-null', () => { + const asset = { exifInfo: { fileSizeInByte: 200, rating: 5, fNumber: 1, description: 'test' } }; + expect(getExifCount(asset as AssetResponseDto)).toBe(4); + }); +}); diff --git a/server/src/utils/exif-utils.ts b/server/src/utils/exif-utils.ts new file mode 100644 index 0000000000..2e11cbdad1 --- /dev/null +++ b/server/src/utils/exif-utils.ts @@ -0,0 +1,6 @@ +import { AssetResponseDto } from "src/dtos/asset-response.dto"; + + +export const getExifCount = (asset: AssetResponseDto) => { + return Object.values(asset.exifInfo ?? {}).filter(Boolean).length; +}; diff --git a/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte b/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte index c6943c6491..fac95056c4 100644 --- a/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte +++ b/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte @@ -11,11 +11,10 @@ import { assetViewingStore } from '$lib/stores/asset-viewing.store'; import { locale } from '$lib/stores/preferences.store'; import { stackAssets } from '$lib/utils/asset-utils'; - import { suggestDuplicate } from '$lib/utils/duplicate-utils'; import { handleError } from '$lib/utils/handle-error'; import type { AssetResponseDto } from '@immich/sdk'; - import { deleteAssets, deleteDuplicates, updateAssets } from '@immich/sdk'; import { Button, HStack, IconButton, modalManager, Text, toastManager } from '@immich/ui'; + import { deDuplicateAll, deleteAssets, keepAll, updateAssets } from '@immich/sdk'; import { mdiCheckOutline, mdiChevronLeft, @@ -56,11 +55,14 @@ ], }; - let duplicates = $state(data.duplicates); + let duplicatesRes = $state(data.duplicatesRes); + // let duplicates = $state(data.duplicates); + let duplicate = $state(data.duplicate); + const { isViewing: showAssetViewer } = assetViewingStore; const correctDuplicatesIndex = (index: number) => { - return Math.max(0, Math.min(index, duplicates.length - 1)); + return Math.max(0, Math.min(index, duplicatesRes.totalItems - 1)); }; let duplicatesIndex = $derived( @@ -71,7 +73,7 @@ })(), ); - let hasDuplicates = $derived(duplicates.length > 0); + let hasDuplicates = $derived(duplicatesRes.totalItems > 0); const withConfirmation = async (callback: () => Promise, prompt?: string, confirmText?: string) => { if (prompt && confirmText) { const isConfirmed = await modalManager.showDialog({ prompt, confirmText }); @@ -104,7 +106,7 @@ await deleteAssets({ assetBulkDeleteDto: { ids: trashIds, force: !featureFlagsManager.value.trash } }); await updateAssets({ assetBulkUpdateDto: { ids: duplicateAssetIds, duplicateId: null } }); - duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId); + // duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId); deletedNotification(trashIds.length); await correctDuplicatesIndexAndGo(duplicatesIndex); @@ -118,39 +120,24 @@ await stackAssets(assets, false); const duplicateAssetIds = assets.map((asset) => asset.id); await updateAssets({ assetBulkUpdateDto: { ids: duplicateAssetIds, duplicateId: null } }); - duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId); + // duplicates = duplicates.filter((duplicate) => duplicate.duplicateId !== duplicateId); await correctDuplicatesIndexAndGo(duplicatesIndex); }; - const handleDeduplicateAll = async () => { - const idsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id); - const idsToDelete = duplicates.flatMap((group, i) => - group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]), - ); - + const handleDeduplicateAll = () => { let prompt, confirmText; - if (featureFlagsManager.value.trash) { - prompt = $t('bulk_trash_duplicates_confirmation', { values: { count: idsToDelete.length } }); + if ($featureFlags.trash) { + prompt = $t('bulk_trash_duplicates_confirmation', { values: { count: 1 } }); confirmText = $t('confirm'); } else { - prompt = $t('bulk_delete_duplicates_confirmation', { values: { count: idsToDelete.length } }); + prompt = $t('bulk_delete_duplicates_confirmation', { values: { count: 1 } }); confirmText = $t('permanently_delete'); } return withConfirmation( async () => { - await deleteAssets({ assetBulkDeleteDto: { ids: idsToDelete, force: !featureFlagsManager.value.trash } }); - await updateAssets({ - assetBulkUpdateDto: { - ids: [...idsToDelete, ...idsToKeep.filter((id): id is string => !!id)], - duplicateId: null, - }, - }); - - duplicates = []; - - deletedNotification(idsToDelete.length); - + await deDuplicateAll(); + deletedNotification(1); page.url.searchParams.delete('index'); await goto(`${AppRoute.DUPLICATES}`); }, @@ -159,19 +146,16 @@ ); }; - const handleKeepAll = async () => { - const ids = duplicates.map(({ duplicateId }) => duplicateId); + const handleKeepAll = () => { return withConfirmation( async () => { - await deleteDuplicates({ bulkIdsDto: { ids } }); - - duplicates = []; + await keepAll(); toastManager.success($t('resolved_all_duplicates')); page.url.searchParams.delete('index'); await goto(`${AppRoute.DUPLICATES}`); }, - $t('bulk_keep_duplicates_confirmation', { values: { count: ids.length } }), + $t('bulk_keep_duplicates_confirmation', { values: { count: 1 } }), $t('confirm'), ); }; @@ -189,7 +173,7 @@ await handlePrevious(); }; const handleNext = async () => { - await correctDuplicatesIndexAndGo(Math.min(duplicatesIndex + 1, duplicates.length - 1)); + await correctDuplicatesIndexAndGo(Math.min(duplicatesIndex + 1, duplicatesRes.totalItems - 1)); }; const handleNextShortcut = async () => { if ($showAssetViewer) { @@ -198,11 +182,14 @@ await handleNext(); }; const handleLast = async () => { - await correctDuplicatesIndexAndGo(duplicates.length - 1); + await correctDuplicatesIndexAndGo(duplicatesRes.totalItems - 1); }; const correctDuplicatesIndexAndGo = async (index: number) => { page.url.searchParams.set('index', correctDuplicatesIndex(index).toString()); await goto(`${AppRoute.DUPLICATES}?${page.url.searchParams.toString()}`); + const result = await data.loadDuplicates(index + 1, 1); + duplicate = result.items[0]; + duplicatesRes = result; }; @@ -213,7 +200,7 @@ ]} /> - + {#snippet buttons()} -

- {duplicatesIndex + 1} / {duplicates.length.toLocaleString($locale)} -

+

{duplicatesIndex + 1}/{duplicatesRes.totalItems.toLocaleString($locale)}

@@ -316,7 +301,7 @@ color="primary" class="flex place-items-center rounded-e-full gap-2 px-2 sm:px-4" onclick={handleLast} - disabled={duplicatesIndex === duplicates.length - 1} + disabled={duplicatesIndex === duplicatesRes.totalItems - 1} > {$t('last')} diff --git a/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.ts b/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.ts index 978f50830e..c20d395b2c 100644 --- a/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.ts +++ b/web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.ts @@ -7,12 +7,24 @@ import type { PageLoad } from './$types'; export const load = (async ({ params, url }) => { await authenticate(url); const asset = await getAssetInfoFromParam(params); - const duplicates = await getAssetDuplicates(); const $t = await getFormatter(); + const indexParam = url.searchParams.get('index') ?? '0'; + const parsedIndex = Number.parseInt(indexParam, 10); + + const duplicates = await getAssetDuplicates({ page: parsedIndex + 1, size: 1 }); + const duplicate = duplicates.items[0]; + + const loadDuplicates = async (newPage: number, newSize: number) => { + return await getAssetDuplicates({ page: newPage, size: newSize }); + }; + return { asset, - duplicates, + duplicatesRes: duplicates, + duplicates: duplicates.items, + duplicate, + loadDuplicates, meta: { title: $t('duplicates'), },