sascha
/
immich
mirror of https://github.com/immich-app/immich.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: postgres reverse geocoding (#5301) 

* feat: add system metadata repository for storing key values for internal usage

* feat: add database entities for geodata

* feat: move reverse geocoding from local-reverse-geocoder to postgresql

* infra: disable synchronization for geodata_places table until typeorm supports earth column

* feat: remove cities override config as we will default all instances to cities500 now

* test: e2e tests don't clear geodata tables on reset
pull/5311/head
Zack Pollard 2023-11-25 18:53:30 +07:00 committed by GitHub
parent 0108211c0f
commit 698226634e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 367 additions and 644 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cli/src/api/open-api/api.ts generated
Unescape Escape View File

@ -1164,22 +1164,6 @@ export interface CheckExistingAssetsResponseDto {
*/
'existingIds': Array<string>;
}
/**
*
* @export
* @enum {string}
*/
export const CitiesFile = {
Cities15000: 'cities15000',
Cities5000: 'cities5000',
Cities1000: 'cities1000',
Cities500: 'cities500'
} as const;
export type CitiesFile = typeof CitiesFile[keyof typeof CitiesFile];
/**
*
* @export
@ -3832,12 +3816,6 @@ export interface SystemConfigPasswordLoginDto {
* @interface SystemConfigReverseGeocodingDto
*/
export interface SystemConfigReverseGeocodingDto {
/**
*
* @type {CitiesFile}
* @memberof SystemConfigReverseGeocodingDto
*/
'citiesFileOverride': CitiesFile;
/**
*
* @type {boolean}
@ -3845,8 +3823,6 @@ export interface SystemConfigReverseGeocodingDto {
*/
'enabled': boolean;
}
/**
*
* @export

3
mobile/openapi/.openapi-generator/FILES generated
Unescape Escape View File

@ -46,7 +46,6 @@ doc/CQMode.md
doc/ChangePasswordDto.md
doc/CheckExistingAssetsDto.md
doc/CheckExistingAssetsResponseDto.md
doc/CitiesFile.md
doc/ClassificationConfig.md
doc/Colorspace.md
doc/CreateAlbumDto.md
@ -231,7 +230,6 @@ lib/model/bulk_ids_dto.dart
lib/model/change_password_dto.dart
lib/model/check_existing_assets_dto.dart
lib/model/check_existing_assets_response_dto.dart
lib/model/cities_file.dart
lib/model/classification_config.dart
lib/model/clip_config.dart
lib/model/clip_mode.dart
@ -388,7 +386,6 @@ test/bulk_ids_dto_test.dart
test/change_password_dto_test.dart
test/check_existing_assets_dto_test.dart
test/check_existing_assets_response_dto_test.dart
test/cities_file_test.dart
test/classification_config_test.dart
test/clip_config_test.dart
test/clip_mode_test.dart

1
mobile/openapi/README.md generated
Unescape Escape View File

@ -244,7 +244,6 @@ Class | Method | HTTP request | Description
- [ChangePasswordDto](doc//ChangePasswordDto.md)
- [CheckExistingAssetsDto](doc//CheckExistingAssetsDto.md)
- [CheckExistingAssetsResponseDto](doc//CheckExistingAssetsResponseDto.md)
- [CitiesFile](doc//CitiesFile.md)
- [ClassificationConfig](doc//ClassificationConfig.md)
- [Colorspace](doc//Colorspace.md)
- [CreateAlbumDto](doc//CreateAlbumDto.md)

14
mobile/openapi/doc/CitiesFile.md generated
Unescape Escape View File

@ -1,14 +0,0 @@
# openapi.model.CitiesFile
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

1
mobile/openapi/doc/SystemConfigReverseGeocodingDto.md generated
Unescape Escape View File

@ -8,7 +8,6 @@ import 'package:openapi/api.dart';
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**citiesFileOverride** | [**CitiesFile**](CitiesFile.md) | |
**enabled** | **bool** | |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

1
mobile/openapi/lib/api.dart generated
Unescape Escape View File

@ -83,7 +83,6 @@ part 'model/cq_mode.dart';
part 'model/change_password_dto.dart';
part 'model/check_existing_assets_dto.dart';
part 'model/check_existing_assets_response_dto.dart';
part 'model/cities_file.dart';
part 'model/classification_config.dart';
part 'model/colorspace.dart';
part 'model/create_album_dto.dart';

2
mobile/openapi/lib/api_client.dart generated
Unescape Escape View File

@ -255,8 +255,6 @@ class ApiClient {
return CheckExistingAssetsDto.fromJson(value);
case 'CheckExistingAssetsResponseDto':
return CheckExistingAssetsResponseDto.fromJson(value);
case 'CitiesFile':
return CitiesFileTypeTransformer().decode(value);
case 'ClassificationConfig':
return ClassificationConfig.fromJson(value);
case 'Colorspace':

3
mobile/openapi/lib/api_helper.dart generated
Unescape Escape View File

@ -73,9 +73,6 @@ String parameterToString(dynamic value) {
if (value is CQMode) {
return CQModeTypeTransformer().encode(value).toString();
}
if (value is CitiesFile) {
return CitiesFileTypeTransformer().encode(value).toString();
}
if (value is Colorspace) {
return ColorspaceTypeTransformer().encode(value).toString();
}

91
mobile/openapi/lib/model/cities_file.dart generated
Unescape Escape View File

@ -1,91 +0,0 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class CitiesFile {
/// Instantiate a new enum with the provided [value].
const CitiesFile._(this.value);
/// The underlying value of this enum member.
final String value;
@override
String toString() => value;
String toJson() => value;
static const cities15000 = CitiesFile._(r'cities15000');
static const cities5000 = CitiesFile._(r'cities5000');
static const cities1000 = CitiesFile._(r'cities1000');
static const cities500 = CitiesFile._(r'cities500');
/// List of all possible values in this [enum][CitiesFile].
static const values = <CitiesFile>[
cities15000,
cities5000,
cities1000,
cities500,
];
static CitiesFile? fromJson(dynamic value) => CitiesFileTypeTransformer().decode(value);
static List<CitiesFile>? listFromJson(dynamic json, {bool growable = false,}) {
final result = <CitiesFile>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = CitiesFile.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
}
/// Transformation class that can [encode] an instance of [CitiesFile] to String,
/// and [decode] dynamic data back to [CitiesFile].
class CitiesFileTypeTransformer {
factory CitiesFileTypeTransformer() => _instance ??= const CitiesFileTypeTransformer._();
const CitiesFileTypeTransformer._();
String encode(CitiesFile data) => data.value;
/// Decodes a [dynamic value][data] to a CitiesFile.
///
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
///
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
/// and users are still using an old app with the old code.
CitiesFile? decode(dynamic data, {bool allowNull = true}) {
if (data != null) {
switch (data) {
case r'cities15000': return CitiesFile.cities15000;
case r'cities5000': return CitiesFile.cities5000;
case r'cities1000': return CitiesFile.cities1000;
case r'cities500': return CitiesFile.cities500;
default:
if (!allowNull) {
throw ArgumentError('Unknown enum value to decode: $data');
}
}
}
return null;
}
/// Singleton [CitiesFileTypeTransformer] instance.
static CitiesFileTypeTransformer? _instance;
}

10
mobile/openapi/lib/model/system_config_reverse_geocoding_dto.dart generated
Unescape Escape View File

@ -13,31 +13,25 @@ part of openapi.api;
class SystemConfigReverseGeocodingDto {
/// Returns a new [SystemConfigReverseGeocodingDto] instance.
SystemConfigReverseGeocodingDto({
required this.citiesFileOverride,
required this.enabled,
});
CitiesFile citiesFileOverride;
bool enabled;
@override
bool operator ==(Object other) => identical(this, other) || other is SystemConfigReverseGeocodingDto &&
other.citiesFileOverride == citiesFileOverride &&
other.enabled == enabled;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(citiesFileOverride.hashCode) +
(enabled.hashCode);
@override
String toString() => 'SystemConfigReverseGeocodingDto[citiesFileOverride=$citiesFileOverride, enabled=$enabled]';
String toString() => 'SystemConfigReverseGeocodingDto[enabled=$enabled]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'citiesFileOverride'] = this.citiesFileOverride;
json[r'enabled'] = this.enabled;
return json;
}
@ -50,7 +44,6 @@ class SystemConfigReverseGeocodingDto {
final json = value.cast<String, dynamic>();
return SystemConfigReverseGeocodingDto(
citiesFileOverride: CitiesFile.fromJson(json[r'citiesFileOverride'])!,
enabled: mapValueOfType<bool>(json, r'enabled')!,
);
}
@ -99,7 +92,6 @@ class SystemConfigReverseGeocodingDto {
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'citiesFileOverride',
'enabled',
};
}

21
mobile/openapi/test/cities_file_test.dart generated
Unescape Escape View File

@ -1,21 +0,0 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for CitiesFile
void main() {
group('test CitiesFile', () {
});
}

5
mobile/openapi/test/system_config_reverse_geocoding_dto_test.dart generated
Unescape Escape View File

@ -16,11 +16,6 @@ void main() {
// final instance = SystemConfigReverseGeocodingDto();
group('test SystemConfigReverseGeocodingDto', () {
// CitiesFile citiesFileOverride
test('to test the property `citiesFileOverride`', () async {
// TODO
});
// bool enabled
test('to test the property `enabled`', () async {
// TODO

2
server/Dockerfile
Unescape Escape View File

@ -31,7 +31,7 @@ COPY --from=prod /usr/src/app/node_modules ./node_modules
COPY --from=prod /usr/src/app/dist ./dist
COPY --from=prod /usr/src/app/bin ./bin
COPY --from=web /usr/src/app/build ./www
COPY server/assets assets
COPY server/resources resources
COPY server/package.json server/package-lock.json ./
COPY server/start*.sh ./
RUN npm link && npm cache clean --force

13
server/immich-openapi-specs.json
Unescape Escape View File

@ -6989,15 +6989,6 @@
],
"type": "object"
},
"CitiesFile": {
"enum": [
"cities15000",
"cities5000",
"cities1000",
"cities500"
],
"type": "string"
},
"ClassificationConfig": {
"properties": {
"enabled": {
@ -9112,15 +9103,11 @@
},
"SystemConfigReverseGeocodingDto": {
"properties": {
"citiesFileOverride": {
"$ref": "#/components/schemas/CitiesFile"
},
"enabled": {
"type": "boolean"
}
},
"required": [
"citiesFileOverride",
"enabled"
],
"type": "object"

264
server/package-lock.json generated
Unescape Escape View File

@ -38,7 +38,6 @@
"i18n-iso-countries": "^7.6.0",
"ioredis": "^5.3.2",
"joi": "^17.10.0",
"local-reverse-geocoder": "0.16.5",
"lodash": "^4.17.21",
"luxon": "^3.4.2",
"mv": "^2.1.1",
@ -4132,18 +4131,6 @@
"node": ">=0.6"
}
},
"node_modules/binary": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
"integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==",
"dependencies": {
"buffers": "~0.1.1",
"chainsaw": "~0.1.0"
},
"engines": {
"node": "*"
}
},
"node_modules/binary-extensions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@ -4329,14 +4316,6 @@
"node": ">=4"
}
},
"node_modules/buffers": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==",
"engines": {
"node": ">=0.2.0"
}
},
"node_modules/buildcheck": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
@ -4500,17 +4479,6 @@
}
]
},
"node_modules/chainsaw": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
"integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==",
"dependencies": {
"traverse": ">=0.3.0 <0.4"
},
"engines": {
"node": "*"
}
},
"node_modules/chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@ -5161,19 +5129,6 @@
"node": ">= 8"
}
},
"node_modules/csv-parse": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz",
"integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw=="
},
"node_modules/data-uri-to-buffer": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
"engines": {
"node": ">= 12"
}
},
"node_modules/date-fns": {
"version": "2.30.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
@ -6300,28 +6255,6 @@
"bser": "2.1.1"
}
},
"node_modules/fetch-blob": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "paypal",
"url": "https://paypal.me/jimmywarting"
}
],
"dependencies": {
"node-domexception": "^1.0.0",
"web-streams-polyfill": "^3.0.3"
},
"engines": {
"node": "^12.20 || >= 14.13"
}
},
"node_modules/figures": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
@ -6575,17 +6508,6 @@
"node": ">= 6"
}
},
"node_modules/formdata-polyfill": {
"version": "4.0.10",
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
"dependencies": {
"fetch-blob": "^3.1.2"
},
"engines": {
"node": ">=12.20.0"
}
},
"node_modules/formidable": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
@ -8323,11 +8245,6 @@
"graceful-fs": "^4.1.6"
}
},
"node_modules/kdt": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/kdt/-/kdt-0.1.0.tgz",
"integrity": "sha512-ueX0gyv7tw4zBq9cQjaCr9qIhGTo5XYHUf/8aUUMHwoyb81KeCZHkSOoUwHGg/mgabvhTKCYjDUuYEmdak6Xjg=="
},
"node_modules/keyv": {
"version": "4.5.3",
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz",
@ -8425,41 +8342,6 @@
"node": ">=6.11.5"
}
},
"node_modules/local-reverse-geocoder": {
"version": "0.16.5",
"resolved": "https://registry.npmjs.org/local-reverse-geocoder/-/local-reverse-geocoder-0.16.5.tgz",
"integrity": "sha512-MgJsyR3s8eeMfRfMvikwIdOG/jh9s78zPPX9kfx1qk5fwQLJnry5Qx5jreclqDPEpjOpNKIqz4aG5BbWGAGLbw==",
"hasInstallScript": true,
"dependencies": {
"async": "^3.2.4",
"csv-parse": "^5.5.0",
"debug": "^4.3.4",
"kdt": "^0.1.0",
"node-fetch": "^3.3.2",
"unzip-stream": "^0.3.1"
},
"engines": {
"node": ">=11.0.0",
"npm": ">=6.4.1"
}
},
"node_modules/local-reverse-geocoder/node_modules/node-fetch": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
"dependencies": {
"data-uri-to-buffer": "^4.0.0",
"fetch-blob": "^3.1.4",
"formdata-polyfill": "^4.0.10"
},
"engines": {
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/node-fetch"
}
},
"node_modules/locate-path": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@ -9077,24 +8959,6 @@
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
"integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
},
"node_modules/node-domexception": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "github",
"url": "https://paypal.me/jimmywarting"
}
],
"engines": {
"node": ">=10.5.0"
}
},
"node_modules/node-emoji": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz",
@ -11717,14 +11581,6 @@
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
},
"node_modules/traverse": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
"integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==",
"engines": {
"node": "*"
}
},
"node_modules/tree-kill": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
@ -12314,15 +12170,6 @@
"node": ">=8"
}
},
"node_modules/unzip-stream": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz",
"integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==",
"dependencies": {
"binary": "^0.3.0",
"mkdirp": "^0.5.1"
}
},
"node_modules/update-browserslist-db": {
"version": "1.0.13",
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz",
@ -12480,14 +12327,6 @@
"defaults": "^1.0.3"
}
},
"node_modules/web-streams-polyfill": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
"engines": {
"node": ">= 8"
}
},
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
@ -15937,15 +15776,6 @@
"integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==",
"dev": true
},
"binary": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
"integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==",
"requires": {
"buffers": "~0.1.1",
"chainsaw": "~0.1.0"
}
},
"binary-extensions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@ -16077,11 +15907,6 @@
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
"integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
},
"buffers": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="
},
"buildcheck": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
@ -16194,14 +16019,6 @@
"integrity": "sha512-UrtAXVcj1mvPBFQ4sKd38daP8dEcXXr5sQe6QNNinaPd0iA/cxg9/l3VrSdL73jgw5sKyuQ6jNgiKO12W3SsVA==",
"dev": true
},
"chainsaw": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
"integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==",
"requires": {
"traverse": ">=0.3.0 <0.4"
}
},
"chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@ -16681,16 +16498,6 @@
"which": "^2.0.1"
}
},
"csv-parse": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz",
"integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw=="
},
"data-uri-to-buffer": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="
},
"date-fns": {
"version": "2.30.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
@ -17523,15 +17330,6 @@
"bser": "2.1.1"
}
},
"fetch-blob": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
"requires": {
"node-domexception": "^1.0.0",
"web-streams-polyfill": "^3.0.3"
}
},
"figures": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
@ -17717,14 +17515,6 @@
"mime-types": "^2.1.12"
}
},
"formdata-polyfill": {
"version": "4.0.10",
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
"requires": {
"fetch-blob": "^3.1.2"
}
},
"formidable": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
@ -19005,11 +18795,6 @@
"universalify": "^2.0.0"
}
},
"kdt": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/kdt/-/kdt-0.1.0.tgz",
"integrity": "sha512-ueX0gyv7tw4zBq9cQjaCr9qIhGTo5XYHUf/8aUUMHwoyb81KeCZHkSOoUwHGg/mgabvhTKCYjDUuYEmdak6Xjg=="
},
"keyv": {
"version": "4.5.3",
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz",
@ -19094,31 +18879,6 @@
"integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==",
"dev": true
},
"local-reverse-geocoder": {
"version": "0.16.5",
"resolved": "https://registry.npmjs.org/local-reverse-geocoder/-/local-reverse-geocoder-0.16.5.tgz",
"integrity": "sha512-MgJsyR3s8eeMfRfMvikwIdOG/jh9s78zPPX9kfx1qk5fwQLJnry5Qx5jreclqDPEpjOpNKIqz4aG5BbWGAGLbw==",
"requires": {
"async": "^3.2.4",
"csv-parse": "^5.5.0",
"debug": "^4.3.4",
"kdt": "^0.1.0",
"node-fetch": "^3.3.2",
"unzip-stream": "^0.3.1"
},
"dependencies": {
"node-fetch": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
"requires": {
"data-uri-to-buffer": "^4.0.0",
"fetch-blob": "^3.1.4",
"formdata-polyfill": "^4.0.10"
}
}
}
},
"locate-path": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@ -19599,11 +19359,6 @@
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
"integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
},
"node-domexception": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="
},
"node-emoji": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz",
@ -21569,11 +21324,6 @@
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
},
"traverse": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
"integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="
},
"tree-kill": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
@ -21900,15 +21650,6 @@
"integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==",
"dev": true
},
"unzip-stream": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz",
"integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==",
"requires": {
"binary": "^0.3.0",
"mkdirp": "^0.5.1"
}
},
"update-browserslist-db": {
"version": "1.0.13",
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz",
@ -22028,11 +21769,6 @@
"defaults": "^1.0.3"
}
},
"web-streams-polyfill": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q=="
},
"webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",

3
server/package.json
Unescape Escape View File

@ -40,6 +40,7 @@
},
"dependencies": {
"@babel/runtime": "^7.22.11",
"@immich/cli": "^2.0.3",
"@nestjs/bullmq": "^10.0.1",
"@nestjs/common": "^10.2.2",
"@nestjs/config": "^3.0.0",
@ -65,10 +66,8 @@
"glob": "^10.3.3",
"handlebars": "^4.7.8",
"i18n-iso-countries": "^7.6.0",
"@immich/cli": "^2.0.3",
"ioredis": "^5.3.2",
"joi": "^17.10.0",
"local-reverse-geocoder": "0.16.5",
"lodash": "^4.17.21",
"luxon": "^3.4.2",
"mv": "^2.1.1",

0
server/assets/style-dark.json → server/resources/style-dark.json
Unescape Escape View File

0
server/assets/style-light.json → server/resources/style-light.json
Unescape Escape View File

41
server/src/domain/metadata/metadata.service.spec.ts
Unescape Escape View File

@ -1,4 +1,4 @@
import { AssetType, CitiesFile, ExifEntity, SystemConfigKey } from '@app/infra/entities';
import { AssetType, ExifEntity, SystemConfigKey } from '@app/infra/entities';
import {
assetStub,
newAlbumRepositoryMock,
@ -15,7 +15,7 @@ import { randomBytes } from 'crypto';
import { Stats } from 'fs';
import { constants } from 'fs/promises';
import { when } from 'jest-when';
import { JobName, QueueName } from '../job';
import { JobName } from '../job';
import {
IAlbumRepository,
IAssetRepository,
@ -78,10 +78,7 @@ describe(MetadataService.name, () => {
describe('init', () => {
beforeEach(async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true },
{ key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_500 },
]);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true }]);
await sut.init();
});
@ -90,42 +87,10 @@ describe(MetadataService.name, () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: false }]);
await sut.init();
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledTimes(1);
expect(metadataMock.init).toHaveBeenCalledTimes(1);
expect(jobMock.resume).toHaveBeenCalledTimes(1);
});
it('should return if deleteCache is false and the cities precision has not changed', async () => {
await sut.init();
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledTimes(1);
expect(metadataMock.init).toHaveBeenCalledTimes(1);
expect(jobMock.resume).toHaveBeenCalledTimes(1);
});
it('should re-init if deleteCache is false but the cities precision has changed', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_1000 },
]);
await sut.init();
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_1000 });
expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
it('should re-init and delete cache if deleteCache is true', async () => {
await sut.init(true);
expect(metadataMock.deleteCache).toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_500 });
expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
});
describe('handleLivePhotoLinking', () => {

22
server/src/domain/metadata/metadata.service.ts
Unescape Escape View File

@ -97,31 +97,24 @@ export class MetadataService {
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
}
async init(deleteCache = false) {
async init() {
if (!this.subscription) {
this.subscription = this.configCore.config$.subscribe(() => this.init());
}
const { reverseGeocoding } = await this.configCore.getConfig();
const { citiesFileOverride } = reverseGeocoding;
const { enabled } = reverseGeocoding;
if (!reverseGeocoding.enabled) {
if (!enabled) {
return;
}
try {
if (deleteCache) {
await this.repository.deleteCache();
} else if (this.oldCities && this.oldCities === citiesFileOverride) {
return;
}
await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
await this.repository.init({ citiesFileOverride });
await this.repository.init();
await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
this.logger.log(`Initialized local reverse geocoder with ${citiesFileOverride}`);
this.oldCities = citiesFileOverride;
this.logger.log(`Initialized local reverse geocoder`);
} catch (error: Error | any) {
this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack);
}
@ -258,8 +251,9 @@ export class MetadataService {
}
try {
const { city, state, country } = await this.repository.reverseGeocode({ latitude, longitude });
Object.assign(exifData, { city, state, country });
const reverseGeocode = await this.repository.reverseGeocode({ latitude, longitude });
if (!reverseGeocode) return;
Object.assign(exifData, reverseGeocode);
} catch (error: Error | any) {
this.logger.warn(
`Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`,

1
server/src/domain/repositories/index.ts
Unescape Escape View File

@ -20,6 +20,7 @@ export * from './shared-link.repository';
export * from './smart-info.repository';
export * from './storage.repository';
export * from './system-config.repository';
export * from './system-metadata.repository';
export * from './tag.repository';
export * from './user-token.repository';
export * from './user.repository';

6
server/src/domain/repositories/metadata.repository.ts
Unescape Escape View File

@ -1,5 +1,4 @@
import { Tags } from 'exiftool-vendored';
import { InitOptions } from 'local-reverse-geocoder';
export const IMetadataRepository = 'IMetadataRepository';
@ -31,9 +30,8 @@ export interface ImmichTags extends Omit<Tags, 'FocalLength' | 'Duration'> {
}
export interface IMetadataRepository {
init(options: Partial<InitOptions>): Promise<void>;
init(): Promise<void>;
teardown(): Promise<void>;
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
deleteCache(): Promise<void>;
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null>;
getExifTags(path: string): Promise<ImmichTags | null>;
}

8
server/src/domain/repositories/system-metadata.repository.ts
Unescape Escape View File

@ -0,0 +1,8 @@
import { SystemMetadata } from '@app/infra/entities';
export const ISystemMetadataRepository = 'ISystemMetadataRepository';
export interface ISystemMetadataRepository {
get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null>;
set<T extends keyof SystemMetadata>(key: T, value: SystemMetadata[T]): Promise<void>;
}

8
server/src/domain/system-config/dto/system-config-reverse-geocoding.dto.ts
Unescape Escape View File

@ -1,12 +1,6 @@
import { CitiesFile } from '@app/infra/entities';
import { ApiProperty } from '@nestjs/swagger';
import { IsBoolean, IsEnum } from 'class-validator';
import { IsBoolean } from 'class-validator';
export class SystemConfigReverseGeocodingDto {
@IsBoolean()
enabled!: boolean;
@IsEnum(CitiesFile)
@ApiProperty({ enum: CitiesFile, enumName: 'CitiesFile' })
citiesFileOverride!: CitiesFile;
}

2
server/src/domain/system-config/system-config.core.ts
Unescape Escape View File

@ -1,6 +1,5 @@
import {
AudioCodec,
CitiesFile,
Colorspace,
CQMode,
SystemConfig,
@ -85,7 +84,6 @@ export const defaults = Object.freeze<SystemConfig>({
},
reverseGeocoding: {
enabled: true,
citiesFileOverride: CitiesFile.CITIES_500,
},
oauth: {
enabled: false,

2
server/src/domain/system-config/system-config.service.spec.ts
Unescape Escape View File

@ -1,6 +1,5 @@
import {
AudioCodec,
CitiesFile,
Colorspace,
CQMode,
SystemConfig,
@ -85,7 +84,6 @@ const updatedConfig = Object.freeze<SystemConfig>({
},
reverseGeocoding: {
enabled: true,
citiesFileOverride: CitiesFile.CITIES_500,
},
oauth: {
autoLaunch: true,

2
server/src/domain/system-config/system-config.service.ts
Unescape Escape View File

@ -79,7 +79,7 @@ export class SystemConfigService {
return this.repository.fetchStyle(styleUrl);
}
return JSON.parse(await this.repository.readFile(`./assets/style-${theme}.json`));
return JSON.parse(await this.repository.readFile(`./resources/style-${theme}.json`));
}
async getCustomCss(): Promise<string> {

10
server/src/infra/entities/geodata-admin1.entity.ts
Unescape Escape View File

@ -0,0 +1,10 @@
import { Column, Entity, PrimaryColumn } from 'typeorm';
@Entity('geodata_admin1')
export class GeodataAdmin1Entity {
@PrimaryColumn({ type: 'varchar' })
key!: string;
@Column({ type: 'varchar' })
name!: string;
}

10
server/src/infra/entities/geodata-admin2.entity.ts
Unescape Escape View File

@ -0,0 +1,10 @@
import { Column, Entity, PrimaryColumn } from 'typeorm';
@Entity('geodata_admin2')
export class GeodataAdmin2Entity {
@PrimaryColumn({ type: 'varchar' })
key!: string;
@Column({ type: 'varchar' })
name!: string;
}

59
server/src/infra/entities/geodata-places.entity.ts
Unescape Escape View File

@ -0,0 +1,59 @@
import { GeodataAdmin1Entity } from '@app/infra/entities/geodata-admin1.entity';
import { GeodataAdmin2Entity } from '@app/infra/entities/geodata-admin2.entity';
import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm';
@Entity('geodata_places', { synchronize: false })
export class GeodataPlacesEntity {
@PrimaryColumn({ type: 'integer' })
id!: number;
@Column({ type: 'varchar', length: 200 })
name!: string;
@Column({ type: 'float' })
longitude!: number;
@Column({ type: 'float' })
latitude!: number;
// @Column({
// generatedType: 'STORED',
// asExpression: 'll_to_earth((latitude)::double precision, (longitude)::double precision)',
// type: 'earth',
// })
earthCoord!: unknown;
@Column({ type: 'char', length: 2 })
countryCode!: string;
@Column({ type: 'varchar', length: 20, nullable: true })
admin1Code!: string;
@Column({ type: 'varchar', length: 80, nullable: true })
admin2Code!: string;
@Column({
type: 'varchar',
generatedType: 'STORED',
asExpression: `"countryCode" || '.' || "admin1Code"`,
nullable: true,
})
admin1Key!: string;
@ManyToOne(() => GeodataAdmin1Entity, { eager: true, nullable: true, createForeignKeyConstraints: false })
admin1!: GeodataAdmin1Entity;
@Column({
type: 'varchar',
generatedType: 'STORED',
asExpression: `"countryCode" || '.' || "admin1Code" || '.' || "admin2Code"`,
nullable: true,
})
admin2Key!: string;
@ManyToOne(() => GeodataAdmin2Entity, { eager: true, nullable: true, createForeignKeyConstraints: false })
admin2!: GeodataAdmin2Entity;
@Column({ type: 'date' })
modificationDate!: Date;
}

12
server/src/infra/entities/index.ts
Unescape Escape View File

@ -1,3 +1,4 @@
import { GeodataAdmin2Entity } from '@app/infra/entities/geodata-admin2.entity';
import { ActivityEntity } from './activity.entity';
import { AlbumEntity } from './album.entity';
import { APIKeyEntity } from './api-key.entity';
@ -6,6 +7,8 @@ import { AssetJobStatusEntity } from './asset-job-status.entity';
import { AssetEntity } from './asset.entity';
import { AuditEntity } from './audit.entity';
import { ExifEntity } from './exif.entity';
import { GeodataAdmin1Entity } from './geodata-admin1.entity';
import { GeodataPlacesEntity } from './geodata-places.entity';
import { LibraryEntity } from './library.entity';
import { MoveEntity } from './move.entity';
import { PartnerEntity } from './partner.entity';
@ -13,6 +16,7 @@ import { PersonEntity } from './person.entity';
import { SharedLinkEntity } from './shared-link.entity';
import { SmartInfoEntity } from './smart-info.entity';
import { SystemConfigEntity } from './system-config.entity';
import { SystemMetadataEntity } from './system-metadata.entity';
import { TagEntity } from './tag.entity';
import { UserTokenEntity } from './user-token.entity';
import { UserEntity } from './user.entity';
@ -25,6 +29,9 @@ export * from './asset-job-status.entity';
export * from './asset.entity';
export * from './audit.entity';
export * from './exif.entity';
export * from './geodata-admin1.entity';
export * from './geodata-admin2.entity';
export * from './geodata-places.entity';
export * from './library.entity';
export * from './move.entity';
export * from './partner.entity';
@ -32,6 +39,7 @@ export * from './person.entity';
export * from './shared-link.entity';
export * from './smart-info.entity';
export * from './system-config.entity';
export * from './system-metadata.entity';
export * from './tag.entity';
export * from './user-token.entity';
export * from './user.entity';
@ -45,12 +53,16 @@ export const databaseEntities = [
AssetJobStatusEntity,
AuditEntity,
ExifEntity,
GeodataPlacesEntity,
GeodataAdmin1Entity,
GeodataAdmin2Entity,
MoveEntity,
PartnerEntity,
PersonEntity,
SharedLinkEntity,
SmartInfoEntity,
SystemConfigEntity,
SystemMetadataEntity,
TagEntity,
UserEntity,
UserTokenEntity,

9
server/src/infra/entities/system-config.entity.ts
Unescape Escape View File

@ -66,7 +66,6 @@ export enum SystemConfigKey {
MAP_DARK_STYLE = 'map.darkStyle',
REVERSE_GEOCODING_ENABLED = 'reverseGeocoding.enabled',
REVERSE_GEOCODING_CITIES_FILE_OVERRIDE = 'reverseGeocoding.citiesFileOverride',
NEW_VERSION_CHECK_ENABLED = 'newVersionCheck.enabled',
@ -145,13 +144,6 @@ export enum Colorspace {
P3 = 'p3',
}
export enum CitiesFile {
CITIES_15000 = 'cities15000',
CITIES_5000 = 'cities5000',
CITIES_1000 = 'cities1000',
CITIES_500 = 'cities500',
}
export interface SystemConfig {
ffmpeg: {
crf: number;
@ -200,7 +192,6 @@ export interface SystemConfig {
};
reverseGeocoding: {
enabled: boolean;
citiesFileOverride: CitiesFile;
};
oauth: {
enabled: boolean;

18
server/src/infra/entities/system-metadata.entity.ts
Unescape Escape View File

@ -0,0 +1,18 @@
import { Column, Entity, PrimaryColumn } from 'typeorm';
@Entity('system_metadata')
export class SystemMetadataEntity {
@PrimaryColumn()
key!: string;
@Column({ type: 'jsonb', default: '{}', transformer: { to: JSON.stringify, from: JSON.parse } })
value!: { [key: string]: unknown };
}
export enum SystemMetadataKey {
REVERSE_GEOCODING_STATE = 'reverse-geocoding-state',
}
export interface SystemMetadata extends Record<SystemMetadataKey, { [key: string]: unknown }> {
[SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
}

3
server/src/infra/infra.config.ts
Unescape Escape View File

@ -74,6 +74,3 @@ function parseTypeSenseConfig(): ConfigurationOptions {
}
export const typesenseConfig: ConfigurationOptions = parseTypeSenseConfig();
export const REVERSE_GEOCODING_DUMP_DIRECTORY =
process.env.REVERSE_GEOCODING_DUMP_DIRECTORY || process.cwd() + '/.reverse-geocoding-dump/';

3
server/src/infra/infra.module.ts
Unescape Escape View File

@ -21,6 +21,7 @@ import {
ISmartInfoRepository,
IStorageRepository,
ISystemConfigRepository,
ISystemMetadataRepository,
ITagRepository,
IUserRepository,
IUserTokenRepository,
@ -56,6 +57,7 @@ import {
SharedLinkRepository,
SmartInfoRepository,
SystemConfigRepository,
SystemMetadataRepository,
TagRepository,
TypesenseRepository,
UserRepository,
@ -84,6 +86,7 @@ const providers: Provider[] = [
{ provide: ISmartInfoRepository, useClass: SmartInfoRepository },
{ provide: IStorageRepository, useClass: FilesystemProvider },
{ provide: ISystemConfigRepository, useClass: SystemConfigRepository },
{ provide: ISystemMetadataRepository, useClass: SystemMetadataRepository },
{ provide: ITagRepository, useClass: TagRepository },
{ provide: IMediaRepository, useClass: MediaRepository },
{ provide: IUserRepository, useClass: UserRepository },

14
server/src/infra/migrations/1700345818045-SystemMetadata.ts
Unescape Escape View File

@ -0,0 +1,14 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class SystemMetadata1700345818045 implements MigrationInterface {
name = 'SystemMetadata1700345818045'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE TABLE "system_metadata" ("key" character varying NOT NULL, "value" jsonb NOT NULL DEFAULT '{}', CONSTRAINT "PK_fa94f6857470fb5b81ec6084465" PRIMARY KEY ("key"))`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "system_metadata"`);
}
}

29
server/src/infra/migrations/1700362016675-Geodata.ts
Unescape Escape View File

@ -0,0 +1,29 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class Geodata1700362016675 implements MigrationInterface {
name = 'Geodata1700362016675'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS cube`)
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS earthdistance`)
await queryRunner.query(`CREATE TABLE "geodata_admin2" ("key" character varying NOT NULL, "name" character varying NOT NULL, CONSTRAINT "PK_1e3886455dbb684d6f6b4756726" PRIMARY KEY ("key"))`);
await queryRunner.query(`CREATE TABLE "geodata_admin1" ("key" character varying NOT NULL, "name" character varying NOT NULL, CONSTRAINT "PK_3fe3a89c5aac789d365871cb172" PRIMARY KEY ("key"))`);
await queryRunner.query(`INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES ($1, $2, $3, $4, $5, $6)`, ["immich","public","geodata_places","GENERATED_COLUMN","admin1Key","\"countryCode\" || '.' || \"admin1Code\""]);
await queryRunner.query(`INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES ($1, $2, $3, $4, $5, $6)`, ["immich","public","geodata_places","GENERATED_COLUMN","admin2Key","\"countryCode\" || '.' || \"admin1Code\" || '.' || \"admin2Code\""]);
await queryRunner.query(`CREATE TABLE "geodata_places" ("id" integer NOT NULL, "name" character varying(200) NOT NULL, "longitude" double precision NOT NULL, "latitude" double precision NOT NULL, "countryCode" character(2) NOT NULL, "admin1Code" character varying(20), "admin2Code" character varying(80), "admin1Key" character varying GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code") STORED, "admin2Key" character varying GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code" || '.' || "admin2Code") STORED, "modificationDate" date NOT NULL, CONSTRAINT "PK_c29918988912ef4036f3d7fbff4" PRIMARY KEY ("id"))`);
await queryRunner.query(`ALTER TABLE "geodata_places" ADD "earthCoord" earth GENERATED ALWAYS AS (ll_to_earth(latitude, longitude)) STORED`)
await queryRunner.query(`CREATE INDEX "IDX_geodata_gist_earthcoord" ON "geodata_places" USING gist ("earthCoord");`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX "IDX_geodata_gist_earthcoord"`);
await queryRunner.query(`DROP TABLE "geodata_places"`);
await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","admin2Key","immich","public","geodata_places"]);
await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","admin1Key","immich","public","geodata_places"]);
await queryRunner.query(`DROP TABLE "geodata_admin1"`);
await queryRunner.query(`DROP TABLE "geodata_admin2"`);
await queryRunner.query(`DROP EXTENSION cube`);
await queryRunner.query(`DROP EXTENSION earthdistance`);
}
}

1
server/src/infra/repositories/index.ts
Unescape Escape View File

@ -19,6 +19,7 @@ export * from './server-info.repository';
export * from './shared-link.repository';
export * from './smart-info.repository';
export * from './system-config.repository';
export * from './system-metadata.repository';
export * from './tag.repository';
export * from './typesense.repository';
export * from './user-token.repository';

209
server/src/infra/repositories/metadata.repository.ts
Unescape Escape View File

@ -1,77 +1,182 @@
import { GeoPoint, IMetadataRepository, ImmichTags, ReverseGeocodeResult } from '@app/domain';
import { REVERSE_GEOCODING_DUMP_DIRECTORY } from '@app/infra';
import { Injectable, Logger } from '@nestjs/common';
import {
GeoPoint,
IMetadataRepository,
ImmichTags,
ISystemMetadataRepository,
ReverseGeocodeResult,
} from '@app/domain';
import { GeodataAdmin1Entity, GeodataAdmin2Entity, GeodataPlacesEntity, SystemMetadataKey } from '@app/infra/entities';
import { DatabaseLock } from '@app/infra/utils/database-locks';
import { Inject, Logger } from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { DefaultReadTaskOptions, exiftool } from 'exiftool-vendored';
import { readdir, rm } from 'fs/promises';
import { createReadStream, existsSync } from 'fs';
import { readFile } from 'fs/promises';
import * as geotz from 'geo-tz';
import { getName } from 'i18n-iso-countries';
import geocoder, { AddressObject, InitOptions } from 'local-reverse-geocoder';
import path from 'path';
import { promisify } from 'util';
export interface AdminCode {
name: string;
asciiName: string;
geoNameId: string;
}
import * as readLine from 'readline';
import { DataSource, DeepPartial, QueryRunner, Repository } from 'typeorm';
export type GeoData = AddressObject & {
admin1Code?: AdminCode | string;
admin2Code?: AdminCode | string;
};
type GeoEntity = GeodataPlacesEntity | GeodataAdmin1Entity | GeodataAdmin2Entity;
type GeoEntityClass = typeof GeodataPlacesEntity | typeof GeodataAdmin1Entity | typeof GeodataAdmin2Entity;
const lookup = promisify<GeoPoint[], number, AddressObject[][]>(geocoder.lookUp).bind(geocoder);
const CITIES_FILE = 'cities500.txt';
@Injectable()
export class MetadataRepository implements IMetadataRepository {
constructor(
@InjectRepository(GeodataPlacesEntity) private readonly geodataPlacesRepository: Repository<GeodataPlacesEntity>,
@InjectRepository(GeodataAdmin1Entity) private readonly geodataAdmin1Repository: Repository<GeodataAdmin1Entity>,
@InjectRepository(GeodataAdmin2Entity) private readonly geodataAdmin2Repository: Repository<GeodataAdmin2Entity>,
@Inject(ISystemMetadataRepository) private readonly systemMetadataRepository: ISystemMetadataRepository,
@InjectDataSource() private dataSource: DataSource,
) {}
private logger = new Logger(MetadataRepository.name);
async init(options: Partial<InitOptions>): Promise<void> {
return new Promise<void>((resolve) => {
geocoder.init(
{
load: {
admin1: true,
admin2: true,
admin3And4: false,
alternateNames: false,
},
countries: [],
dumpDirectory: REVERSE_GEOCODING_DUMP_DIRECTORY,
...options,
},
resolve,
);
async init(): Promise<void> {
this.logger.log('Initializing metadata repository');
const geodataDate = await readFile('/usr/src/resources/geodata-date.txt', 'utf8');
await this.geodataPlacesRepository.query('SELECT pg_advisory_lock($1)', [DatabaseLock.GeodataImport]);
const geocodingMetadata = await this.systemMetadataRepository.get(SystemMetadataKey.REVERSE_GEOCODING_STATE);
if (geocodingMetadata?.lastUpdate === geodataDate) {
await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]);
return;
}
this.logger.log('Importing geodata to database from file');
const queryRunner = this.dataSource.createQueryRunner();
await queryRunner.connect();
try {
await queryRunner.startTransaction();
await this.loadCities500(queryRunner);
await this.loadAdmin1(queryRunner);
await this.loadAdmin2(queryRunner);
await queryRunner.commitTransaction();
} catch (e) {
this.logger.fatal('Error importing geodata', e);
await queryRunner.rollbackTransaction();
throw e;
} finally {
await queryRunner.release();
}
await this.systemMetadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, {
lastUpdate: geodataDate,
lastImportFileName: CITIES_FILE,
});
}
async teardown() {
await exiftool.end();
await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]);
this.logger.log('Geodata import completed');
}
async deleteCache() {
const dumpDirectory = REVERSE_GEOCODING_DUMP_DIRECTORY;
if (dumpDirectory) {
// delete contents
const items = await readdir(dumpDirectory, { withFileTypes: true });
const folders = items.filter((item) => item.isDirectory());
for (const { name } of folders) {
await rm(path.join(dumpDirectory, name), { recursive: true, force: true });
private async loadGeodataToTableFromFile<T extends GeoEntity>(
queryRunner: QueryRunner,
lineToEntityMapper: (lineSplit: string[]) => T,
filePath: string,
entity: GeoEntityClass,
) {
if (!existsSync(filePath)) {
this.logger.error(`Geodata file ${filePath} not found`);
throw new Error(`Geodata file ${filePath} not found`);
}
await queryRunner.manager.clear(entity);
const input = createReadStream(filePath);
let buffer: DeepPartial<T>[] = [];
const lineReader = readLine.createInterface({ input: input });
for await (const line of lineReader) {
const lineSplit = line.split('\t');
buffer.push(lineToEntityMapper(lineSplit));
if (buffer.length > 1000) {
await queryRunner.manager.save(buffer);
buffer = [];
}
}
await queryRunner.manager.save(buffer);
}
private async loadCities500(queryRunner: QueryRunner) {
await this.loadGeodataToTableFromFile<GeodataPlacesEntity>(
queryRunner,
(lineSplit: string[]) =>
this.geodataPlacesRepository.create({
id: parseInt(lineSplit[0]),
name: lineSplit[1],
latitude: parseFloat(lineSplit[4]),
longitude: parseFloat(lineSplit[5]),
countryCode: lineSplit[8],
admin1Code: lineSplit[10],
admin2Code: lineSplit[11],
modificationDate: lineSplit[18],
}),
`/usr/src/resources/${CITIES_FILE}`,
GeodataPlacesEntity,
);
}
private async loadAdmin1(queryRunner: QueryRunner) {
await this.loadGeodataToTableFromFile<GeodataAdmin1Entity>(
queryRunner,
(lineSplit: string[]) =>
this.geodataAdmin1Repository.create({
key: lineSplit[0],
name: lineSplit[1],
}),
'/usr/src/resources/admin1CodesASCII.txt',
GeodataAdmin1Entity,
);
}
private async loadAdmin2(queryRunner: QueryRunner) {
await this.loadGeodataToTableFromFile<GeodataAdmin2Entity>(
queryRunner,
(lineSplit: string[]) =>
this.geodataAdmin2Repository.create({
key: lineSplit[0],
name: lineSplit[1],
}),
'/usr/src/resources/admin2Codes.txt',
GeodataAdmin2Entity,
);
}
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {
async teardown() {
await exiftool.end();
}
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null> {
this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
const [address] = await lookup([point], 1);
this.logger.verbose(`Raw: ${JSON.stringify(address, null, 2)}`);
const response = await this.geodataPlacesRepository
.createQueryBuilder('geoplaces')
.leftJoinAndSelect('geoplaces.admin1', 'admin1')
.leftJoinAndSelect('geoplaces.admin2', 'admin2')
.where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point)
.orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")')
.limit(1)
.getOne();
if (!response) {
this.logger.warn(
`Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
);
return null;
}
this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`);
const { countryCode, name: city, admin1Code, admin2Code } = address[0] as GeoData;
const { countryCode, name: city, admin1, admin2 } = response;
const country = getName(countryCode, 'en') ?? null;
const stateParts = [(admin2Code as AdminCode)?.name, (admin1Code as AdminCode)?.name].filter((name) => !!name);
const stateParts = [admin2?.name, admin1?.name].filter((name) => !!name);
const state = stateParts.length > 0 ? stateParts.join(', ') : null;
this.logger.debug(`Normalized: ${JSON.stringify({ country, state, city })}`);
return { country, state, city };
}

20
server/src/infra/repositories/system-metadata.repository.ts
Unescape Escape View File

@ -0,0 +1,20 @@
import { ISystemMetadataRepository } from '@app/domain/repositories/system-metadata.repository';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { SystemMetadata, SystemMetadataEntity } from '../entities';
export class SystemMetadataRepository implements ISystemMetadataRepository {
constructor(
@InjectRepository(SystemMetadataEntity)
private repository: Repository<SystemMetadataEntity>,
) {}
async get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null> {
const metadata = await this.repository.findOne({ where: { key } });
if (!metadata) return null;
return metadata.value as SystemMetadata[T];
}
async set<T extends keyof SystemMetadata>(key: T, value: SystemMetadata[T]): Promise<void> {
await this.repository.upsert({ key, value }, { conflictPaths: { key: true } });
}
}

3
server/src/infra/utils/database-locks.ts
Unescape Escape View File

@ -0,0 +1,3 @@
export enum DatabaseLock {
GeodataImport = 100,
}

10
server/src/microservices/app.service.ts
Unescape Escape View File

@ -92,16 +92,6 @@ export class AppService {
[JobName.LIBRARY_QUEUE_CLEANUP]: () => this.libraryService.handleQueueCleanup(),
});
process.on('uncaughtException', async (error: Error | any) => {
const isCsvError = error.code === 'CSV_RECORD_INCONSISTENT_FIELDS_LENGTH';
if (!isCsvError) {
throw error;
}
this.logger.warn('Geocoding csv parse error, trying again without cache...');
await this.metadataService.init(true);
});
await this.metadataService.init();
await this.searchService.init();
}

1
server/test/repositories/metadata.repository.mock.ts
Unescape Escape View File

@ -2,7 +2,6 @@ import { IMetadataRepository } from '@app/domain';
export const newMetadataRepositoryMock = (): jest.Mocked<IMetadataRepository> => {
return {
deleteCache: jest.fn(),
getExifTags: jest.fn(),
init: jest.fn(),
teardown: jest.fn(),

4
server/test/test-utils.ts
Unescape Escape View File

@ -25,7 +25,9 @@ export const db = {
const tableNames =
entities.length > 0
? entities.map((entity) => em.getRepository(entity).metadata.tableName)
: dataSource.entityMetadatas.map((entity) => entity.tableName);
: dataSource.entityMetadatas
.map((entity) => entity.tableName)
.filter((tableName) => !tableName.startsWith('geodata'));
let deleteUsers = false;
for (const tableName of tableNames) {

24
web/src/api/open-api/api.ts generated
Unescape Escape View File

@ -1164,22 +1164,6 @@ export interface CheckExistingAssetsResponseDto {
*/
'existingIds': Array<string>;
}
/**
*
* @export
* @enum {string}
*/
export const CitiesFile = {
Cities15000: 'cities15000',
Cities5000: 'cities5000',
Cities1000: 'cities1000',
Cities500: 'cities500'
} as const;
export type CitiesFile = typeof CitiesFile[keyof typeof CitiesFile];
/**
*
* @export
@ -3832,12 +3816,6 @@ export interface SystemConfigPasswordLoginDto {
* @interface SystemConfigReverseGeocodingDto
*/
export interface SystemConfigReverseGeocodingDto {
/**
*
* @type {CitiesFile}
* @memberof SystemConfigReverseGeocodingDto
*/
'citiesFileOverride': CitiesFile;
/**
*
* @type {boolean}
@ -3845,8 +3823,6 @@ export interface SystemConfigReverseGeocodingDto {
*/
'enabled': boolean;
}
/**
*
* @export

22
web/src/lib/components/admin-page/settings/map-settings/map-settings.svelte
Unescape Escape View File

@ -4,13 +4,12 @@
NotificationType,
} from '$lib/components/shared-components/notification/notification';
import { handleError } from '$lib/utils/handle-error';
import { api, CitiesFile, SystemConfigDto } from '@api';
import { api, SystemConfigDto } from '@api';
import { cloneDeep, isEqual } from 'lodash-es';
import { fade } from 'svelte/transition';
import SettingAccordion from '../setting-accordion.svelte';
import SettingButtonsRow from '../setting-buttons-row.svelte';
import SettingSwitch from '../setting-switch.svelte';
import SettingSelect from '../setting-select.svelte';
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
export let config: SystemConfigDto; // this is the config that is being edited
@ -39,7 +38,6 @@
},
reverseGeocoding: {
enabled: config.reverseGeocoding.enabled,
citiesFileOverride: config.reverseGeocoding.citiesFileOverride,
},
},
});
@ -131,24 +129,6 @@
subtitle="Enable reverse geocoding"
bind:checked={config.reverseGeocoding.enabled}
/>
<hr />
<SettingSelect
label="Precision"
desc="Set reverse geocoding precision"
name="reverse-geocoding-precision"
bind:value={config.reverseGeocoding.citiesFileOverride}
options={[
{ value: CitiesFile.Cities500, text: 'Cities with more than 500 people' },
{ value: CitiesFile.Cities1000, text: 'Cities with more than 1000 people' },
{ value: CitiesFile.Cities5000, text: 'Cities with more than 5000 people' },
{ value: CitiesFile.Cities15000, text: 'Cities with more than 15000 people' },
]}
disabled={disabled || !config.reverseGeocoding.enabled}
isEdited={config.reverseGeocoding.citiesFileOverride !==
savedConfig.reverseGeocoding.citiesFileOverride}
/>
</div></SettingAccordion
>