feat(server): optimize get asset query (#7176)

* faster query

* add index

* remove legacy code

* update mock

* remove unused imports

* add relations

* add stack

* formatting

* remove stack relation

* remove unused import

* increase chunk size

* generate sql

* linting

* fix typing

* formatting

mobile/lib/shared/services/asset.service.dart

@@ -63,7 +63,7 @@ class AssetService {
 
   /// Returns `null` if the server state did not change, else list of assets
   Future<List<Asset>?> _getRemoteAssets(User user) async {
-    const int chunkSize = 5000;
+    const int chunkSize = 10000;
     try {
       final DateTime now = DateTime.now().toUtc();
       final List<Asset> allAssets = [];

server/src/domain/repositories/asset.repository.ts

@@ -1,4 +1,4 @@
-import { AssetSearchOptions, SearchExploreItem } from '@app/domain';
+import { AssetSearchOneToOneRelationOptions, AssetSearchOptions, SearchExploreItem } from '@app/domain';
 import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '@app/infra/entities';
 import { FindOptionsRelations, FindOptionsSelect } from 'typeorm';
 import { Paginated, PaginationOptions } from '../domain.util';
@@ -133,6 +133,10 @@ export interface IAssetRepository {
   getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
   deleteAll(ownerId: string): Promise<void>;
   getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
+  getAllByFileCreationDate(
+    pagination: PaginationOptions,
+    options?: AssetSearchOneToOneRelationOptions,
+  ): Paginated<AssetEntity>;
   getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
   updateAll(ids: string[], options: Partial<AssetEntity>): Promise<void>;
   save(asset: Pick<AssetEntity, 'id'> & Partial<AssetEntity>): Promise<AssetEntity>;

server/src/domain/repositories/search.repository.ts

@@ -69,7 +69,6 @@ export interface SearchAssetIDOptions {
 export interface SearchUserIdOptions {
   deviceId?: string;
   libraryId?: string;
-  ownerId?: string;
   userIds?: string[];
 }
 
@@ -147,16 +146,19 @@ export interface SearchPaginationOptions {
   size: number;
 }
 
-export type AssetSearchOptions = SearchDateOptions &
+type BaseAssetSearchOptions = SearchDateOptions &
   SearchIdOptions &
   SearchExifOptions &
   SearchOrderOptions &
   SearchPathOptions &
-  SearchRelationOptions &
   SearchStatusOptions &
   SearchUserIdOptions &
   SearchPeopleOptions;
 
+export type AssetSearchOptions = BaseAssetSearchOptions & SearchRelationOptions;
+
+export type AssetSearchOneToOneRelationOptions = BaseAssetSearchOptions & SearchOneToOneRelationOptions;
+
 export type AssetSearchBuilderOptions = Omit<AssetSearchOptions, 'orderDirection'>;
 
 export type SmartSearchOptions = SearchDateOptions &

server/src/immich/api-v1/asset/asset-repository.ts

@@ -1,10 +1,8 @@
-import { AssetEntity, ExifEntity } from '@app/infra/entities';
-import { OptionalBetween } from '@app/infra/infra.utils';
+import { AssetEntity } from '@app/infra/entities';
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { In } from 'typeorm/find-options/operator/In.js';
 import { Repository } from 'typeorm/repository/Repository.js';
-import { AssetSearchDto } from './dto/asset-search.dto';
 import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
 import { SearchPropertiesDto } from './dto/search-properties.dto';
 import { CuratedLocationsResponseDto } from './response-dto/curated-locations-response.dto';
@@ -21,7 +19,6 @@ export interface AssetOwnerCheck extends AssetCheck {
 
 export interface IAssetRepositoryV1 {
   get(id: string): Promise<AssetEntity | null>;
-  getAllByUserId(userId: string, dto: AssetSearchDto): Promise<AssetEntity[]>;
   getLocationsByUserId(userId: string): Promise<CuratedLocationsResponseDto[]>;
   getDetectedObjectsByUserId(userId: string): Promise<CuratedObjectsResponseDto[]>;
   getSearchPropertiesByUserId(userId: string): Promise<SearchPropertiesDto[]>;
@@ -34,10 +31,7 @@ export const IAssetRepositoryV1 = 'IAssetRepositoryV1';
 
 @Injectable()
 export class AssetRepositoryV1 implements IAssetRepositoryV1 {
-  constructor(
-    @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
-    @InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
-  ) {}
+  constructor(@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>) {}
 
   getSearchPropertiesByUserId(userId: string): Promise<SearchPropertiesDto[]> {
     return this.assetRepository
@@ -89,33 +83,6 @@ export class AssetRepositoryV1 implements IAssetRepositoryV1 {
     );
   }
 
-  /**
-   * Get all assets belong to the user on the database
-   * @param ownerId
-   */
-  getAllByUserId(ownerId: string, dto: AssetSearchDto): Promise<AssetEntity[]> {
-    return this.assetRepository.find({
-      where: {
-        ownerId,
-        isVisible: true,
-        isFavorite: dto.isFavorite,
-        isArchived: dto.isArchived,
-        updatedAt: OptionalBetween(dto.updatedAfter, dto.updatedBefore),
-      },
-      relations: {
-        exifInfo: true,
-        tags: true,
-        stack: { assets: true },
-      },
-      skip: dto.skip || 0,
-      take: dto.take,
-      order: {
-        fileCreatedAt: 'DESC',
-      },
-      withDeleted: true,
-    });
-  }
-
   get(id: string): Promise<AssetEntity | null> {
     return this.assetRepository.findOne({
       where: { id },

server/src/immich/api-v1/asset/asset.service.spec.ts

@@ -77,7 +77,6 @@ describe('AssetService', () => {
   beforeEach(() => {
     assetRepositoryMockV1 = {
       get: jest.fn(),
-      getAllByUserId: jest.fn(),
       getDetectedObjectsByUserId: jest.fn(),
       getLocationsByUserId: jest.fn(),
       getSearchPropertiesByUserId: jest.fn(),

server/src/immich/api-v1/asset/asset.service.ts

@@ -114,8 +114,11 @@ export class AssetService {
   public async getAllAssets(auth: AuthDto, dto: AssetSearchDto): Promise<AssetResponseDto[]> {
     const userId = dto.userId || auth.user.id;
     await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
-    const assets = await this.assetRepositoryV1.getAllByUserId(userId, dto);
-    return assets.map((asset) => mapAsset(asset, { withStack: true }));
+    const assets = await this.assetRepository.getAllByFileCreationDate(
+      { take: dto.take ?? 1000, skip: dto.skip },
+      { ...dto, userIds: [userId], withDeleted: true, orderDirection: 'DESC', withExif: true },
+    );
+    return assets.items.map((asset) => mapAsset(asset));
   }
 
   async serveThumbnail(auth: AuthDto, assetId: string, dto: GetAssetThumbnailDto): Promise<ImmichFileResponse> {

server/src/infra/entities/asset.entity.ts

@@ -85,6 +85,7 @@ export class AssetEntity {
   @DeleteDateColumn({ type: 'timestamptz', nullable: true })
   deletedAt!: Date | null;
 
+  @Index('idx_asset_file_created_at')
   @Column({ type: 'timestamptz' })
   fileCreatedAt!: Date;
 

server/src/infra/migrations/1708227417898-AddFileCreatedAtIndex.ts

@@ -0,0 +1,12 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddFileCreatedAtIndex1708227417898 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE INDEX idx_asset_file_created_at ON assets ("fileCreatedAt")`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX idx_asset_file_created_at`);
+    }
+}

server/src/infra/repositories/asset.repository.ts

@@ -2,6 +2,7 @@ import {
   AssetBuilderOptions,
   AssetCreate,
   AssetExploreFieldOptions,
+  AssetSearchOneToOneRelationOptions,
   AssetSearchOptions,
   AssetStats,
   AssetStatsOptions,
@@ -175,8 +176,12 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
-  getByUserId(pagination: PaginationOptions, userId: string, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
-    return this.getAll(pagination, { ...options, ownerId: userId });
+  getByUserId(
+    pagination: PaginationOptions,
+    userId: string,
+    options: Omit<AssetSearchOptions, 'userIds'> = {},
+  ): Paginated<AssetEntity> {
+    return this.getAll(pagination, { ...options, userIds: [userId] });
   }
 
   @GenerateSql({ params: [[DummyValue.UUID]] })
@@ -205,6 +210,29 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
+  @GenerateSql({
+    params: [
+      { skip: 20_000, take: 10_000 },
+      {
+        takenBefore: DummyValue.DATE,
+        userIds: [DummyValue.UUID],
+      },
+    ],
+  })
+  getAllByFileCreationDate(
+    pagination: PaginationOptions,
+    options: AssetSearchOneToOneRelationOptions = {},
+  ): Paginated<AssetEntity> {
+    let builder = this.repository.createQueryBuilder('asset');
+    builder = searchAssetBuilder(builder, options);
+    builder.orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC');
+    return paginatedBuilder<AssetEntity>(builder, {
+      mode: PaginationMode.LIMIT_OFFSET,
+      skip: pagination.skip,
+      take: pagination.take,
+    });
+  }
+
   /**
    * Get assets by device's Id on the database
    * @param ownerId

server/src/infra/sql/asset.repository.sql

@@ -395,6 +395,55 @@ ORDER BY
 LIMIT
   1
 
+-- AssetRepository.getAllByFileCreationDate
+SELECT
+  "asset"."id" AS "asset_id",
+  "asset"."deviceAssetId" AS "asset_deviceAssetId",
+  "asset"."ownerId" AS "asset_ownerId",
+  "asset"."libraryId" AS "asset_libraryId",
+  "asset"."deviceId" AS "asset_deviceId",
+  "asset"."type" AS "asset_type",
+  "asset"."originalPath" AS "asset_originalPath",
+  "asset"."resizePath" AS "asset_resizePath",
+  "asset"."webpPath" AS "asset_webpPath",
+  "asset"."thumbhash" AS "asset_thumbhash",
+  "asset"."encodedVideoPath" AS "asset_encodedVideoPath",
+  "asset"."createdAt" AS "asset_createdAt",
+  "asset"."updatedAt" AS "asset_updatedAt",
+  "asset"."deletedAt" AS "asset_deletedAt",
+  "asset"."fileCreatedAt" AS "asset_fileCreatedAt",
+  "asset"."localDateTime" AS "asset_localDateTime",
+  "asset"."fileModifiedAt" AS "asset_fileModifiedAt",
+  "asset"."isFavorite" AS "asset_isFavorite",
+  "asset"."isArchived" AS "asset_isArchived",
+  "asset"."isExternal" AS "asset_isExternal",
+  "asset"."isReadOnly" AS "asset_isReadOnly",
+  "asset"."isOffline" AS "asset_isOffline",
+  "asset"."checksum" AS "asset_checksum",
+  "asset"."duration" AS "asset_duration",
+  "asset"."isVisible" AS "asset_isVisible",
+  "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
+  "asset"."originalFileName" AS "asset_originalFileName",
+  "asset"."sidecarPath" AS "asset_sidecarPath",
+  "asset"."stackId" AS "asset_stackId"
+FROM
+  "assets" "asset"
+WHERE
+  (
+    "asset"."fileCreatedAt" <= $1
+    AND 1 = 1
+    AND "asset"."ownerId" IN ($2)
+    AND 1 = 1
+    AND 1 = 1
+  )
+  AND ("asset"."deletedAt" IS NULL)
+ORDER BY
+  "asset"."fileCreatedAt" DESC
+LIMIT
+  10001
+OFFSET
+  20000
+
 -- AssetRepository.getAllByDeviceId
 SELECT
   "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",

server/test/repositories/asset.repository.mock.ts

@@ -18,6 +18,7 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
     getFirstAssetForAlbumId: jest.fn(),
     getLastUpdatedAssetForAlbumId: jest.fn(),
     getAll: jest.fn().mockResolvedValue({ items: [], hasNextPage: false }),
+    getAllByFileCreationDate: jest.fn(),
     getAllByDeviceId: jest.fn(),
     updateAll: jest.fn(),
     getByLibraryId: jest.fn(),