fix(server): "view all" for cities only showing 12 cities (#8035)

* view all cities

* increase limit

* rename endpoint

* optimize query

* remove pagination

* update sql

* linting

* revert sort by count in explore page for now

* fix query

* fix

* update sql

* move to search, add partner support

* update sql

* pr feedback

* euphemism

* parameters as separate variable

* move comment

* update sql

* linting

mobile/openapi/README.md

@@ -161,6 +161,7 @@ Class | Method | HTTP request | Description
 *PersonApi* | [**reassignFaces**](doc//PersonApi.md#reassignfaces) | **PUT** /person/{id}/reassign | 
 *PersonApi* | [**updatePeople**](doc//PersonApi.md#updatepeople) | **PUT** /person | 
 *PersonApi* | [**updatePerson**](doc//PersonApi.md#updateperson) | **PUT** /person/{id} | 
+*SearchApi* | [**getAssetsByCity**](doc//SearchApi.md#getassetsbycity) | **GET** /search/cities | 
 *SearchApi* | [**getExploreData**](doc//SearchApi.md#getexploredata) | **GET** /search/explore | 
 *SearchApi* | [**getSearchSuggestions**](doc//SearchApi.md#getsearchsuggestions) | **GET** /search/suggestions | 
 *SearchApi* | [**search**](doc//SearchApi.md#search) | **GET** /search | 

mobile/openapi/doc/SearchApi.md

@@ -9,6 +9,7 @@ All URIs are relative to */api*
 
 Method | HTTP request | Description
 ------------- | ------------- | -------------
+[**getAssetsByCity**](SearchApi.md#getassetsbycity) | **GET** /search/cities | 
 [**getExploreData**](SearchApi.md#getexploredata) | **GET** /search/explore | 
 [**getSearchSuggestions**](SearchApi.md#getsearchsuggestions) | **GET** /search/suggestions | 
 [**search**](SearchApi.md#search) | **GET** /search | 
@@ -18,6 +19,57 @@ Method | HTTP request | Description
 [**searchSmart**](SearchApi.md#searchsmart) | **POST** /search/smart | 
 
 
+# **getAssetsByCity**
+> List<AssetResponseDto> getAssetsByCity()
+
+
+
+### Example
+```dart
+import 'package:openapi/api.dart';
+// TODO Configure API key authorization: cookie
+//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
+// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
+//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
+// TODO Configure API key authorization: api_key
+//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
+// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
+//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
+// TODO Configure HTTP Bearer authorization: bearer
+// Case 1. Use String Token
+//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
+// Case 2. Use Function which generate token.
+// String yourTokenGeneratorFunction() { ... }
+//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
+
+final api_instance = SearchApi();
+
+try {
+    final result = api_instance.getAssetsByCity();
+    print(result);
+} catch (e) {
+    print('Exception when calling SearchApi->getAssetsByCity: $e\n');
+}
+```
+
+### Parameters
+This endpoint does not need any parameter.
+
+### Return type
+
+[**List<AssetResponseDto>**](AssetResponseDto.md)
+
+### Authorization
+
+[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
+
+### HTTP request headers
+
+ - **Content-Type**: Not defined
+ - **Accept**: application/json
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
 # **getExploreData**
 > List<SearchExploreResponseDto> getExploreData()
 

mobile/openapi/lib/api/search_api.dart

@@ -16,6 +16,50 @@ class SearchApi {
 
   final ApiClient apiClient;
 
+  /// Performs an HTTP 'GET /search/cities' operation and returns the [Response].
+  Future<Response> getAssetsByCityWithHttpInfo() async {
+    // ignore: prefer_const_declarations
+    final path = r'/search/cities';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      path,
+      'GET',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  Future<List<AssetResponseDto>?> getAssetsByCity() async {
+    final response = await getAssetsByCityWithHttpInfo();
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+    // When a remote server returns no body with a status of 204, we shall not decode it.
+    // At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
+    // FormatException when trying to decode an empty string.
+    if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
+      final responseBody = await _decodeBodyBytes(response);
+      return (await apiClient.deserializeAsync(responseBody, 'List<AssetResponseDto>') as List)
+        .cast<AssetResponseDto>()
+        .toList(growable: false);
+
+    }
+    return null;
+  }
+
   /// Performs an HTTP 'GET /search/explore' operation and returns the [Response].
   Future<Response> getExploreDataWithHttpInfo() async {
     // ignore: prefer_const_declarations

mobile/openapi/test/search_api_test.dart

@@ -17,6 +17,11 @@ void main() {
   // final instance = SearchApi();
 
   group('tests for SearchApi', () {
+    //Future<List<AssetResponseDto>> getAssetsByCity() async
+    test('test getAssetsByCity', () async {
+      // TODO
+    });
+
     //Future<List<SearchExploreResponseDto>> getExploreData() async
     test('test getExploreData', () async {
       // TODO

open-api/immich-openapi-specs.json

@@ -4597,6 +4597,41 @@
         ]
       }
     },
+    "/search/cities": {
+      "get": {
+        "operationId": "getAssetsByCity",
+        "parameters": [],
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "items": {
+                    "$ref": "#/components/schemas/AssetResponseDto"
+                  },
+                  "type": "array"
+                }
+              }
+            },
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Search"
+        ]
+      }
+    },
     "/search/explore": {
       "get": {
         "operationId": "getExploreData",

open-api/typescript-sdk/src/fetch-client.ts

@@ -2204,6 +2204,14 @@ export function search({ clip, motion, page, q, query, recent, size, smart, $typ
         ...opts
     }));
 }
+export function getAssetsByCity(opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchJson<{
+        status: 200;
+        data: AssetResponseDto[];
+    }>("/search/cities", {
+        ...opts
+    }));
+}
 export function getExploreData(opts?: Oazapfts.RequestOpts) {
     return oazapfts.ok(oazapfts.fetchJson<{
         status: 200;

server/src/domain/repositories/search.repository.ts

@@ -187,5 +187,6 @@ export interface ISearchRepository {
   searchFaces(search: FaceEmbeddingSearch): Promise<FaceSearchResult[]>;
   upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void>;
   searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]>;
+  getAssetsByCity(userIds: string[]): Promise<AssetEntity[]>;
   deleteAllSearchEmbeddings(): Promise<void>;
 }

server/src/domain/search/search.service.ts

@@ -115,6 +115,32 @@ export class SearchService {
     return this.mapResponse(items, hasNextPage ? (page + 1).toString() : null);
   }
 
+  async getAssetsByCity(auth: AuthDto): Promise<AssetResponseDto[]> {
+    const userIds = await this.getUserIdsToSearch(auth);
+    const assets = await this.searchRepository.getAssetsByCity(userIds);
+    return assets.map((asset) => mapAsset(asset));
+  }
+
+  getSearchSuggestions(auth: AuthDto, dto: SearchSuggestionRequestDto): Promise<string[]> {
+    switch (dto.type) {
+      case SearchSuggestionType.COUNTRY: {
+        return this.metadataRepository.getCountries(auth.user.id);
+      }
+      case SearchSuggestionType.STATE: {
+        return this.metadataRepository.getStates(auth.user.id, dto.country);
+      }
+      case SearchSuggestionType.CITY: {
+        return this.metadataRepository.getCities(auth.user.id, dto.country, dto.state);
+      }
+      case SearchSuggestionType.CAMERA_MAKE: {
+        return this.metadataRepository.getCameraMakes(auth.user.id, dto.model);
+      }
+      case SearchSuggestionType.CAMERA_MODEL: {
+        return this.metadataRepository.getCameraModels(auth.user.id, dto.make);
+      }
+    }
+  }
+
   // TODO: remove after implementing new search filters
   /** @deprecated */
   async search(auth: AuthDto, dto: SearchDto): Promise<SearchResponseDto> {
@@ -191,24 +217,4 @@ export class SearchService {
       },
     };
   }
-
-  async getSearchSuggestions(auth: AuthDto, dto: SearchSuggestionRequestDto): Promise<string[]> {
-    switch (dto.type) {
-      case SearchSuggestionType.COUNTRY: {
-        return this.metadataRepository.getCountries(auth.user.id);
-      }
-      case SearchSuggestionType.STATE: {
-        return this.metadataRepository.getStates(auth.user.id, dto.country);
-      }
-      case SearchSuggestionType.CITY: {
-        return this.metadataRepository.getCities(auth.user.id, dto.country, dto.state);
-      }
-      case SearchSuggestionType.CAMERA_MAKE: {
-        return this.metadataRepository.getCameraMakes(auth.user.id, dto.model);
-      }
-      case SearchSuggestionType.CAMERA_MODEL: {
-        return this.metadataRepository.getCameraModels(auth.user.id, dto.make);
-      }
-    }
-  }
 }

server/src/immich/controllers/search.controller.ts

@@ -1,4 +1,5 @@
 import {
+  AssetResponseDto,
   AuthDto,
   MetadataSearchDto,
   PersonResponseDto,
@@ -55,6 +56,11 @@ export class SearchController {
     return this.service.searchPlaces(dto);
   }
 
+  @Get('cities')
+  getAssetsByCity(@Auth() auth: AuthDto): Promise<AssetResponseDto[]> {
+    return this.service.getAssetsByCity(auth);
+  }
+
   @Get('suggestions')
   getSearchSuggestions(@Auth() auth: AuthDto, @Query() dto: SearchSuggestionRequestDto): Promise<string[]> {
     return this.service.getSearchSuggestions(auth, dto);

server/src/infra/repositories/search.repository.ts

@@ -15,6 +15,7 @@ import { getCLIPModelInfo } from '@app/domain/smart-info/smart-info.constant';
 import {
   AssetEntity,
   AssetFaceEntity,
+  AssetType,
   GeodataPlacesEntity,
   SmartInfoEntity,
   SmartSearchEntity,
@@ -33,6 +34,7 @@ import { Instrumentation } from '../instrumentation';
 export class SearchRepository implements ISearchRepository {
   private logger = new ImmichLogger(SearchRepository.name);
   private faceColumns: string[];
+  private assetsByCityQuery: string;
 
   constructor(
     @InjectRepository(SmartInfoEntity) private repository: Repository<SmartInfoEntity>,
@@ -45,6 +47,14 @@ export class SearchRepository implements ISearchRepository {
       .getMetadata(AssetFaceEntity)
       .ownColumns.map((column) => column.propertyName)
       .filter((propertyName) => propertyName !== 'embedding');
+    this.assetsByCityQuery =
+      assetsByCityCte +
+      this.assetRepository
+        .createQueryBuilder('asset')
+        .innerJoinAndSelect('asset.exifInfo', 'exif')
+        .withDeleted()
+        .getQuery() +
+      ' INNER JOIN cte ON asset.id = cte."assetId"';
   }
 
   async init(modelName: string): Promise<void> {
@@ -220,6 +230,27 @@ export class SearchRepository implements ISearchRepository {
       .getMany();
   }
 
+  @GenerateSql({ params: [[DummyValue.UUID]] })
+  async getAssetsByCity(userIds: string[]): Promise<AssetEntity[]> {
+    const parameters = [userIds.join(', '), true, false, AssetType.IMAGE];
+    const rawRes = await this.repository.query(this.assetsByCityQuery, parameters);
+
+    const items: AssetEntity[] = [];
+    for (const res of rawRes) {
+      const item = { exifInfo: {} as Record<string, any> } as Record<string, any>;
+      for (const [key, value] of Object.entries(res)) {
+        if (key.startsWith('exif_')) {
+          item.exifInfo[key.replace('exif_', '')] = value;
+        } else {
+          item[key.replace('asset_', '')] = value;
+        }
+      }
+      items.push(item as AssetEntity);
+    }
+
+    return items;
+  }
+
   async upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void> {
     await this.repository.upsert(smartInfo, { conflictPaths: ['assetId'] });
     if (!smartInfo.assetId || !embedding) {
@@ -290,3 +321,30 @@ export class SearchRepository implements ISearchRepository {
     return runtimeConfig;
   }
 }
+
+// the performance difference between this and the normal way is too huge to ignore, e.g. 3s vs 4ms
+const assetsByCityCte = `
+WITH RECURSIVE cte AS (
+  (
+    SELECT city, "assetId"
+    FROM exif
+    INNER JOIN assets ON exif."assetId" = assets.id
+    WHERE "ownerId" IN ($1) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4
+    ORDER BY city
+    LIMIT 1
+  )
+
+  UNION ALL
+  
+  SELECT l.city, l."assetId"
+  FROM cte c
+    , LATERAL (
+    SELECT city, "assetId"
+    FROM exif
+    INNER JOIN assets ON exif."assetId" = assets.id
+    WHERE city > c.city AND "ownerId" IN ($1) AND "isVisible" = $2 AND "isArchived" = $3 AND type = $4
+    ORDER BY city
+    LIMIT 1
+    ) l
+)
+`;

server/src/infra/sql/search.repository.sql

@@ -266,3 +266,111 @@ ORDER BY
   ) ASC
 LIMIT
   20
+
+-- SearchRepository.getAssetsByCity
+WITH RECURSIVE
+  cte AS (
+    (
+      SELECT
+        city,
+        "assetId"
+      FROM
+        exif
+        INNER JOIN assets ON exif."assetId" = assets.id
+      WHERE
+        "ownerId" IN ($1)
+        AND "isVisible" = $2
+        AND "isArchived" = $3
+        AND type = $4
+      ORDER BY
+        city
+      LIMIT
+        1
+    )
+    UNION ALL
+    SELECT
+      l.city,
+      l."assetId"
+    FROM
+      cte c,
+      LATERAL (
+        SELECT
+          city,
+          "assetId"
+        FROM
+          exif
+          INNER JOIN assets ON exif."assetId" = assets.id
+        WHERE
+          city > c.city
+          AND "ownerId" IN ($1)
+          AND "isVisible" = $2
+          AND "isArchived" = $3
+          AND type = $4
+        ORDER BY
+          city
+        LIMIT
+          1
+      ) l
+  )
+SELECT
+  "asset"."id" AS "asset_id",
+  "asset"."deviceAssetId" AS "asset_deviceAssetId",
+  "asset"."ownerId" AS "asset_ownerId",
+  "asset"."libraryId" AS "asset_libraryId",
+  "asset"."deviceId" AS "asset_deviceId",
+  "asset"."type" AS "asset_type",
+  "asset"."originalPath" AS "asset_originalPath",
+  "asset"."resizePath" AS "asset_resizePath",
+  "asset"."webpPath" AS "asset_webpPath",
+  "asset"."thumbhash" AS "asset_thumbhash",
+  "asset"."encodedVideoPath" AS "asset_encodedVideoPath",
+  "asset"."createdAt" AS "asset_createdAt",
+  "asset"."updatedAt" AS "asset_updatedAt",
+  "asset"."deletedAt" AS "asset_deletedAt",
+  "asset"."fileCreatedAt" AS "asset_fileCreatedAt",
+  "asset"."localDateTime" AS "asset_localDateTime",
+  "asset"."fileModifiedAt" AS "asset_fileModifiedAt",
+  "asset"."isFavorite" AS "asset_isFavorite",
+  "asset"."isArchived" AS "asset_isArchived",
+  "asset"."isExternal" AS "asset_isExternal",
+  "asset"."isReadOnly" AS "asset_isReadOnly",
+  "asset"."isOffline" AS "asset_isOffline",
+  "asset"."checksum" AS "asset_checksum",
+  "asset"."duration" AS "asset_duration",
+  "asset"."isVisible" AS "asset_isVisible",
+  "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
+  "asset"."originalFileName" AS "asset_originalFileName",
+  "asset"."sidecarPath" AS "asset_sidecarPath",
+  "asset"."stackId" AS "asset_stackId",
+  "exif"."assetId" AS "exif_assetId",
+  "exif"."description" AS "exif_description",
+  "exif"."exifImageWidth" AS "exif_exifImageWidth",
+  "exif"."exifImageHeight" AS "exif_exifImageHeight",
+  "exif"."fileSizeInByte" AS "exif_fileSizeInByte",
+  "exif"."orientation" AS "exif_orientation",
+  "exif"."dateTimeOriginal" AS "exif_dateTimeOriginal",
+  "exif"."modifyDate" AS "exif_modifyDate",
+  "exif"."timeZone" AS "exif_timeZone",
+  "exif"."latitude" AS "exif_latitude",
+  "exif"."longitude" AS "exif_longitude",
+  "exif"."projectionType" AS "exif_projectionType",
+  "exif"."city" AS "exif_city",
+  "exif"."livePhotoCID" AS "exif_livePhotoCID",
+  "exif"."autoStackId" AS "exif_autoStackId",
+  "exif"."state" AS "exif_state",
+  "exif"."country" AS "exif_country",
+  "exif"."make" AS "exif_make",
+  "exif"."model" AS "exif_model",
+  "exif"."lensModel" AS "exif_lensModel",
+  "exif"."fNumber" AS "exif_fNumber",
+  "exif"."focalLength" AS "exif_focalLength",
+  "exif"."iso" AS "exif_iso",
+  "exif"."exposureTime" AS "exif_exposureTime",
+  "exif"."profileDescription" AS "exif_profileDescription",
+  "exif"."colorspace" AS "exif_colorspace",
+  "exif"."bitsPerSample" AS "exif_bitsPerSample",
+  "exif"."fps" AS "exif_fps"
+FROM
+  "assets" "asset"
+  INNER JOIN "exif" "exif" ON "exif"."assetId" = "asset"."id"
+  INNER JOIN cte ON asset.id = cte."assetId"

server/test/repositories/search.repository.mock.ts

@@ -8,6 +8,7 @@ export const newSearchRepositoryMock = (): jest.Mocked<ISearchRepository> => {
     searchFaces: jest.fn(),
     upsert: jest.fn(),
     searchPlaces: jest.fn(),
+    getAssetsByCity: jest.fn(),
     deleteAllSearchEmbeddings: jest.fn(),
   };
 };

web/src/routes/(user)/places/+page.svelte

@@ -3,20 +3,20 @@
   import Icon from '$lib/components/elements/icon.svelte';
   import UserPageLayout from '$lib/components/layouts/user-page-layout.svelte';
   import { AppRoute } from '$lib/constants';
-  import type { SearchExploreResponseDto } from '@immich/sdk';
   import { mdiMapMarkerOff } from '@mdi/js';
   import type { PageData } from './$types';
   import { getMetadataSearchQuery } from '$lib/utils/metadata-search';
+  import type { AssetResponseDto } from '@immich/sdk';
 
   export let data: PageData;
 
-  const CITY_FIELD = 'exifInfo.city';
-  const getFieldItems = (items: SearchExploreResponseDto[]) => {
-    const targetField = items.find((item) => item.fieldName === CITY_FIELD);
-    return targetField?.items || [];
+  type AssetWithCity = AssetResponseDto & {
+    exifInfo: {
+      city: string;
+    };
   };
 
-  $: places = getFieldItems(data.items);
+  $: places = data.items.filter((item): item is AssetWithCity => !!item.exifInfo?.city);
   $: hasPlaces = places.length > 0;
 
   let innerHeight: number;
@@ -27,17 +27,18 @@
 <UserPageLayout title="Places">
   {#if hasPlaces}
     <div class="flex flex-row flex-wrap gap-4">
-      {#each places as item (item.data.id)}
-        <a class="relative" href="{AppRoute.SEARCH}?{getMetadataSearchQuery({ city: item.value })}" draggable="false">
+      {#each places as item (item.id)}
+        {@const city = item.exifInfo.city}
+        <a class="relative" href="{AppRoute.SEARCH}?{getMetadataSearchQuery({ city })}" draggable="false">
           <div
             class="flex w-[calc((100vw-(72px+5rem))/2)] max-w-[156px] justify-center overflow-hidden rounded-xl brightness-75 filter"
           >
-            <Thumbnail thumbnailSize={156} asset={item.data} readonly />
+            <Thumbnail thumbnailSize={156} asset={item} readonly />
           </div>
           <span
             class="w-100 absolute bottom-2 w-full text-ellipsis px-1 text-center text-sm font-medium capitalize text-white backdrop-blur-[1px] hover:cursor-pointer"
           >
-            {item.value}
+            {city}
           </span>
         </a>
       {/each}

web/src/routes/(user)/places/+page.ts

@@ -1,10 +1,10 @@
 import { authenticate } from '$lib/utils/auth';
-import { getExploreData } from '@immich/sdk';
+import { getAssetsByCity } from '@immich/sdk';
 import type { PageLoad } from './$types';
 
 export const load = (async () => {
   await authenticate();
-  const items = await getExploreData();
+  const items = await getAssetsByCity();
 
   return {
     items,