mirror of https://github.com/immich-app/immich.git
feat(server): dynamic job concurrency (#2622)
* feat(server): dynamic job concurrency * styling and add setting info to top of the job list * regenerate api * remove DETECT_OBJECT job --------- Co-authored-by: Alex Tran <alex.tran1502@gmail.com>pull/2634/head
Jason Rasmussen
GitHub
parent
656dc08406
commit
2493dfaba3
@ -0,0 +1,15 @@
|
|||||||
|
# openapi.model.JobSettingsDto
|
||||||
|
|
||||||
|
## Load the model package
|
||||||
|
```dart
|
||||||
|
import 'package:openapi/api.dart';
|
||||||
|
```
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
Name | Type | Description | Notes
|
||||||
|
------------ | ------------- | ------------- | -------------
|
||||||
|
**concurrency** | **int** | |
|
||||||
|
|
||||||
|
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,24 @@
|
|||||||
|
# openapi.model.SystemConfigJobDto
|
||||||
|
|
||||||
|
## Load the model package
|
||||||
|
```dart
|
||||||
|
import 'package:openapi/api.dart';
|
||||||
|
```
|
||||||
|
|
||||||
|
## Properties
|
||||||
|
Name | Type | Description | Notes
|
||||||
|
------------ | ------------- | ------------- | -------------
|
||||||
|
**thumbnailGeneration** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
**metadataExtraction** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
**videoConversion** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
**objectTagging** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
**clipEncoding** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
**storageTemplateMigration** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
**backgroundTask** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
**search** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
**recognizeFaces** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
**sidecar** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||||
|
|
||||||
|
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,109 @@
|
|||||||
|
//
|
||||||
|
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||||
|
//
|
||||||
|
// @dart=2.12
|
||||||
|
|
||||||
|
// ignore_for_file: unused_element, unused_import
|
||||||
|
// ignore_for_file: always_put_required_named_parameters_first
|
||||||
|
// ignore_for_file: constant_identifier_names
|
||||||
|
// ignore_for_file: lines_longer_than_80_chars
|
||||||
|
|
||||||
|
part of openapi.api;
|
||||||
|
|
||||||
|
class JobSettingsDto {
|
||||||
|
/// Returns a new [JobSettingsDto] instance.
|
||||||
|
JobSettingsDto({
|
||||||
|
required this.concurrency,
|
||||||
|
});
|
||||||
|
|
||||||
|
int concurrency;
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool operator ==(Object other) => identical(this, other) || other is JobSettingsDto &&
|
||||||
|
other.concurrency == concurrency;
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get hashCode =>
|
||||||
|
// ignore: unnecessary_parenthesis
|
||||||
|
(concurrency.hashCode);
|
||||||
|
|
||||||
|
@override
|
||||||
|
String toString() => 'JobSettingsDto[concurrency=$concurrency]';
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() {
|
||||||
|
final json = <String, dynamic>{};
|
||||||
|
json[r'concurrency'] = this.concurrency;
|
||||||
|
return json;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a new [JobSettingsDto] instance and imports its values from
|
||||||
|
/// [value] if it's a [Map], null otherwise.
|
||||||
|
// ignore: prefer_constructors_over_static_methods
|
||||||
|
static JobSettingsDto? fromJson(dynamic value) {
|
||||||
|
if (value is Map) {
|
||||||
|
final json = value.cast<String, dynamic>();
|
||||||
|
|
||||||
|
// Ensure that the map contains the required keys.
|
||||||
|
// Note 1: the values aren't checked for validity beyond being non-null.
|
||||||
|
// Note 2: this code is stripped in release mode!
|
||||||
|
assert(() {
|
||||||
|
requiredKeys.forEach((key) {
|
||||||
|
assert(json.containsKey(key), 'Required key "JobSettingsDto[$key]" is missing from JSON.');
|
||||||
|
assert(json[key] != null, 'Required key "JobSettingsDto[$key]" has a null value in JSON.');
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}());
|
||||||
|
|
||||||
|
return JobSettingsDto(
|
||||||
|
concurrency: mapValueOfType<int>(json, r'concurrency')!,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
static List<JobSettingsDto> listFromJson(dynamic json, {bool growable = false,}) {
|
||||||
|
final result = <JobSettingsDto>[];
|
||||||
|
if (json is List && json.isNotEmpty) {
|
||||||
|
for (final row in json) {
|
||||||
|
final value = JobSettingsDto.fromJson(row);
|
||||||
|
if (value != null) {
|
||||||
|
result.add(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result.toList(growable: growable);
|
||||||
|
}
|
||||||
|
|
||||||
|
static Map<String, JobSettingsDto> mapFromJson(dynamic json) {
|
||||||
|
final map = <String, JobSettingsDto>{};
|
||||||
|
if (json is Map && json.isNotEmpty) {
|
||||||
|
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
|
||||||
|
for (final entry in json.entries) {
|
||||||
|
final value = JobSettingsDto.fromJson(entry.value);
|
||||||
|
if (value != null) {
|
||||||
|
map[entry.key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
// maps a json object with a list of JobSettingsDto-objects as value to a dart map
|
||||||
|
static Map<String, List<JobSettingsDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
|
||||||
|
final map = <String, List<JobSettingsDto>>{};
|
||||||
|
if (json is Map && json.isNotEmpty) {
|
||||||
|
// ignore: parameter_assignments
|
||||||
|
json = json.cast<String, dynamic>();
|
||||||
|
for (final entry in json.entries) {
|
||||||
|
map[entry.key] = JobSettingsDto.listFromJson(entry.value, growable: growable,);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The list of required keys that must be present in a JSON.
|
||||||
|
static const requiredKeys = <String>{
|
||||||
|
'concurrency',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
@ -0,0 +1,181 @@
|
|||||||
|
//
|
||||||
|
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||||
|
//
|
||||||
|
// @dart=2.12
|
||||||
|
|
||||||
|
// ignore_for_file: unused_element, unused_import
|
||||||
|
// ignore_for_file: always_put_required_named_parameters_first
|
||||||
|
// ignore_for_file: constant_identifier_names
|
||||||
|
// ignore_for_file: lines_longer_than_80_chars
|
||||||
|
|
||||||
|
part of openapi.api;
|
||||||
|
|
||||||
|
class SystemConfigJobDto {
|
||||||
|
/// Returns a new [SystemConfigJobDto] instance.
|
||||||
|
SystemConfigJobDto({
|
||||||
|
required this.thumbnailGeneration,
|
||||||
|
required this.metadataExtraction,
|
||||||
|
required this.videoConversion,
|
||||||
|
required this.objectTagging,
|
||||||
|
required this.clipEncoding,
|
||||||
|
required this.storageTemplateMigration,
|
||||||
|
required this.backgroundTask,
|
||||||
|
required this.search,
|
||||||
|
required this.recognizeFaces,
|
||||||
|
required this.sidecar,
|
||||||
|
});
|
||||||
|
|
||||||
|
JobSettingsDto thumbnailGeneration;
|
||||||
|
|
||||||
|
JobSettingsDto metadataExtraction;
|
||||||
|
|
||||||
|
JobSettingsDto videoConversion;
|
||||||
|
|
||||||
|
JobSettingsDto objectTagging;
|
||||||
|
|
||||||
|
JobSettingsDto clipEncoding;
|
||||||
|
|
||||||
|
JobSettingsDto storageTemplateMigration;
|
||||||
|
|
||||||
|
JobSettingsDto backgroundTask;
|
||||||
|
|
||||||
|
JobSettingsDto search;
|
||||||
|
|
||||||
|
JobSettingsDto recognizeFaces;
|
||||||
|
|
||||||
|
JobSettingsDto sidecar;
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool operator ==(Object other) => identical(this, other) || other is SystemConfigJobDto &&
|
||||||
|
other.thumbnailGeneration == thumbnailGeneration &&
|
||||||
|
other.metadataExtraction == metadataExtraction &&
|
||||||
|
other.videoConversion == videoConversion &&
|
||||||
|
other.objectTagging == objectTagging &&
|
||||||
|
other.clipEncoding == clipEncoding &&
|
||||||
|
other.storageTemplateMigration == storageTemplateMigration &&
|
||||||
|
other.backgroundTask == backgroundTask &&
|
||||||
|
other.search == search &&
|
||||||
|
other.recognizeFaces == recognizeFaces &&
|
||||||
|
other.sidecar == sidecar;
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get hashCode =>
|
||||||
|
// ignore: unnecessary_parenthesis
|
||||||
|
(thumbnailGeneration.hashCode) +
|
||||||
|
(metadataExtraction.hashCode) +
|
||||||
|
(videoConversion.hashCode) +
|
||||||
|
(objectTagging.hashCode) +
|
||||||
|
(clipEncoding.hashCode) +
|
||||||
|
(storageTemplateMigration.hashCode) +
|
||||||
|
(backgroundTask.hashCode) +
|
||||||
|
(search.hashCode) +
|
||||||
|
(recognizeFaces.hashCode) +
|
||||||
|
(sidecar.hashCode);
|
||||||
|
|
||||||
|
@override
|
||||||
|
String toString() => 'SystemConfigJobDto[thumbnailGeneration=$thumbnailGeneration, metadataExtraction=$metadataExtraction, videoConversion=$videoConversion, objectTagging=$objectTagging, clipEncoding=$clipEncoding, storageTemplateMigration=$storageTemplateMigration, backgroundTask=$backgroundTask, search=$search, recognizeFaces=$recognizeFaces, sidecar=$sidecar]';
|
||||||
|
|
||||||
|
Map<String, dynamic> toJson() {
|
||||||
|
final json = <String, dynamic>{};
|
||||||
|
json[r'thumbnailGeneration'] = this.thumbnailGeneration;
|
||||||
|
json[r'metadataExtraction'] = this.metadataExtraction;
|
||||||
|
json[r'videoConversion'] = this.videoConversion;
|
||||||
|
json[r'objectTagging'] = this.objectTagging;
|
||||||
|
json[r'clipEncoding'] = this.clipEncoding;
|
||||||
|
json[r'storageTemplateMigration'] = this.storageTemplateMigration;
|
||||||
|
json[r'backgroundTask'] = this.backgroundTask;
|
||||||
|
json[r'search'] = this.search;
|
||||||
|
json[r'recognizeFaces'] = this.recognizeFaces;
|
||||||
|
json[r'sidecar'] = this.sidecar;
|
||||||
|
return json;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns a new [SystemConfigJobDto] instance and imports its values from
|
||||||
|
/// [value] if it's a [Map], null otherwise.
|
||||||
|
// ignore: prefer_constructors_over_static_methods
|
||||||
|
static SystemConfigJobDto? fromJson(dynamic value) {
|
||||||
|
if (value is Map) {
|
||||||
|
final json = value.cast<String, dynamic>();
|
||||||
|
|
||||||
|
// Ensure that the map contains the required keys.
|
||||||
|
// Note 1: the values aren't checked for validity beyond being non-null.
|
||||||
|
// Note 2: this code is stripped in release mode!
|
||||||
|
assert(() {
|
||||||
|
requiredKeys.forEach((key) {
|
||||||
|
assert(json.containsKey(key), 'Required key "SystemConfigJobDto[$key]" is missing from JSON.');
|
||||||
|
assert(json[key] != null, 'Required key "SystemConfigJobDto[$key]" has a null value in JSON.');
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
}());
|
||||||
|
|
||||||
|
return SystemConfigJobDto(
|
||||||
|
thumbnailGeneration: JobSettingsDto.fromJson(json[r'thumbnailGeneration'])!,
|
||||||
|
metadataExtraction: JobSettingsDto.fromJson(json[r'metadataExtraction'])!,
|
||||||
|
videoConversion: JobSettingsDto.fromJson(json[r'videoConversion'])!,
|
||||||
|
objectTagging: JobSettingsDto.fromJson(json[r'objectTagging'])!,
|
||||||
|
clipEncoding: JobSettingsDto.fromJson(json[r'clipEncoding'])!,
|
||||||
|
storageTemplateMigration: JobSettingsDto.fromJson(json[r'storageTemplateMigration'])!,
|
||||||
|
backgroundTask: JobSettingsDto.fromJson(json[r'backgroundTask'])!,
|
||||||
|
search: JobSettingsDto.fromJson(json[r'search'])!,
|
||||||
|
recognizeFaces: JobSettingsDto.fromJson(json[r'recognizeFaces'])!,
|
||||||
|
sidecar: JobSettingsDto.fromJson(json[r'sidecar'])!,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
static List<SystemConfigJobDto> listFromJson(dynamic json, {bool growable = false,}) {
|
||||||
|
final result = <SystemConfigJobDto>[];
|
||||||
|
if (json is List && json.isNotEmpty) {
|
||||||
|
for (final row in json) {
|
||||||
|
final value = SystemConfigJobDto.fromJson(row);
|
||||||
|
if (value != null) {
|
||||||
|
result.add(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result.toList(growable: growable);
|
||||||
|
}
|
||||||
|
|
||||||
|
static Map<String, SystemConfigJobDto> mapFromJson(dynamic json) {
|
||||||
|
final map = <String, SystemConfigJobDto>{};
|
||||||
|
if (json is Map && json.isNotEmpty) {
|
||||||
|
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
|
||||||
|
for (final entry in json.entries) {
|
||||||
|
final value = SystemConfigJobDto.fromJson(entry.value);
|
||||||
|
if (value != null) {
|
||||||
|
map[entry.key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
// maps a json object with a list of SystemConfigJobDto-objects as value to a dart map
|
||||||
|
static Map<String, List<SystemConfigJobDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
|
||||||
|
final map = <String, List<SystemConfigJobDto>>{};
|
||||||
|
if (json is Map && json.isNotEmpty) {
|
||||||
|
// ignore: parameter_assignments
|
||||||
|
json = json.cast<String, dynamic>();
|
||||||
|
for (final entry in json.entries) {
|
||||||
|
map[entry.key] = SystemConfigJobDto.listFromJson(entry.value, growable: growable,);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The list of required keys that must be present in a JSON.
|
||||||
|
static const requiredKeys = <String>{
|
||||||
|
'thumbnailGeneration',
|
||||||
|
'metadataExtraction',
|
||||||
|
'videoConversion',
|
||||||
|
'objectTagging',
|
||||||
|
'clipEncoding',
|
||||||
|
'storageTemplateMigration',
|
||||||
|
'backgroundTask',
|
||||||
|
'search',
|
||||||
|
'recognizeFaces',
|
||||||
|
'sidecar',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
@ -0,0 +1,27 @@
|
|||||||
|
//
|
||||||
|
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||||
|
//
|
||||||
|
// @dart=2.12
|
||||||
|
|
||||||
|
// ignore_for_file: unused_element, unused_import
|
||||||
|
// ignore_for_file: always_put_required_named_parameters_first
|
||||||
|
// ignore_for_file: constant_identifier_names
|
||||||
|
// ignore_for_file: lines_longer_than_80_chars
|
||||||
|
|
||||||
|
import 'package:openapi/api.dart';
|
||||||
|
import 'package:test/test.dart';
|
||||||
|
|
||||||
|
// tests for JobSettingsDto
|
||||||
|
void main() {
|
||||||
|
// final instance = JobSettingsDto();
|
||||||
|
|
||||||
|
group('test JobSettingsDto', () {
|
||||||
|
// int concurrency
|
||||||
|
test('to test the property `concurrency`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,72 @@
|
|||||||
|
//
|
||||||
|
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||||
|
//
|
||||||
|
// @dart=2.12
|
||||||
|
|
||||||
|
// ignore_for_file: unused_element, unused_import
|
||||||
|
// ignore_for_file: always_put_required_named_parameters_first
|
||||||
|
// ignore_for_file: constant_identifier_names
|
||||||
|
// ignore_for_file: lines_longer_than_80_chars
|
||||||
|
|
||||||
|
import 'package:openapi/api.dart';
|
||||||
|
import 'package:test/test.dart';
|
||||||
|
|
||||||
|
// tests for SystemConfigJobDto
|
||||||
|
void main() {
|
||||||
|
// final instance = SystemConfigJobDto();
|
||||||
|
|
||||||
|
group('test SystemConfigJobDto', () {
|
||||||
|
// JobSettingsDto thumbnailGeneration
|
||||||
|
test('to test the property `thumbnailGeneration`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
// JobSettingsDto metadataExtraction
|
||||||
|
test('to test the property `metadataExtraction`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
// JobSettingsDto videoConversion
|
||||||
|
test('to test the property `videoConversion`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
// JobSettingsDto objectTagging
|
||||||
|
test('to test the property `objectTagging`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
// JobSettingsDto clipEncoding
|
||||||
|
test('to test the property `clipEncoding`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
// JobSettingsDto storageTemplateMigration
|
||||||
|
test('to test the property `storageTemplateMigration`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
// JobSettingsDto backgroundTask
|
||||||
|
test('to test the property `backgroundTask`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
// JobSettingsDto search
|
||||||
|
test('to test the property `search`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
// JobSettingsDto recognizeFaces
|
||||||
|
test('to test the property `recognizeFaces`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
// JobSettingsDto sidecar
|
||||||
|
test('to test the property `sidecar`', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,75 @@
|
|||||||
|
import {
|
||||||
|
FacialRecognitionService,
|
||||||
|
IDeleteFilesJob,
|
||||||
|
JobName,
|
||||||
|
JobService,
|
||||||
|
MediaService,
|
||||||
|
MetadataService,
|
||||||
|
PersonService,
|
||||||
|
SearchService,
|
||||||
|
SmartInfoService,
|
||||||
|
StorageService,
|
||||||
|
StorageTemplateService,
|
||||||
|
SystemConfigService,
|
||||||
|
UserService,
|
||||||
|
} from '@app/domain';
|
||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class AppService {
|
||||||
|
constructor(
|
||||||
|
// TODO refactor to domain
|
||||||
|
private metadataProcessor: MetadataExtractionProcessor,
|
||||||
|
|
||||||
|
private facialRecognitionService: FacialRecognitionService,
|
||||||
|
private jobService: JobService,
|
||||||
|
private mediaService: MediaService,
|
||||||
|
private metadataService: MetadataService,
|
||||||
|
private personService: PersonService,
|
||||||
|
private searchService: SearchService,
|
||||||
|
private smartInfoService: SmartInfoService,
|
||||||
|
private storageTemplateService: StorageTemplateService,
|
||||||
|
private storageService: StorageService,
|
||||||
|
private systemConfigService: SystemConfigService,
|
||||||
|
private userService: UserService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async init() {
|
||||||
|
await this.jobService.registerHandlers({
|
||||||
|
[JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
|
||||||
|
[JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(),
|
||||||
|
[JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data),
|
||||||
|
[JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data),
|
||||||
|
[JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data),
|
||||||
|
[JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data),
|
||||||
|
[JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data),
|
||||||
|
[JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(),
|
||||||
|
[JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(),
|
||||||
|
[JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(),
|
||||||
|
[JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data),
|
||||||
|
[JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data),
|
||||||
|
[JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data),
|
||||||
|
[JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data),
|
||||||
|
[JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data),
|
||||||
|
[JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
|
||||||
|
[JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
|
||||||
|
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
|
||||||
|
[JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
|
||||||
|
[JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
|
||||||
|
[JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),
|
||||||
|
[JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data),
|
||||||
|
[JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data),
|
||||||
|
[JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),
|
||||||
|
[JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data),
|
||||||
|
[JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data),
|
||||||
|
[JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data),
|
||||||
|
[JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data),
|
||||||
|
[JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data),
|
||||||
|
[JobName.PERSON_CLEANUP]: () => this.personService.handlePersonCleanup(),
|
||||||
|
[JobName.QUEUE_SIDECAR]: (data) => this.metadataService.handleQueueSidecar(data),
|
||||||
|
[JobName.SIDECAR_DISCOVERY]: (data) => this.metadataService.handleSidecarDiscovery(data),
|
||||||
|
[JobName.SIDECAR_SYNC]: () => this.metadataService.handleSidecarSync(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,113 +0,0 @@
|
|||||||
import {
|
|
||||||
FacialRecognitionService,
|
|
||||||
IDeleteFilesJob,
|
|
||||||
JobItem,
|
|
||||||
JobName,
|
|
||||||
JobService,
|
|
||||||
JOBS_TO_QUEUE,
|
|
||||||
MediaService,
|
|
||||||
MetadataService,
|
|
||||||
PersonService,
|
|
||||||
QueueName,
|
|
||||||
QUEUE_TO_CONCURRENCY,
|
|
||||||
SearchService,
|
|
||||||
SmartInfoService,
|
|
||||||
StorageService,
|
|
||||||
StorageTemplateService,
|
|
||||||
SystemConfigService,
|
|
||||||
UserService,
|
|
||||||
} from '@app/domain';
|
|
||||||
import { getQueueToken } from '@nestjs/bull';
|
|
||||||
import { Injectable, Logger } from '@nestjs/common';
|
|
||||||
import { ModuleRef } from '@nestjs/core';
|
|
||||||
import { Queue } from 'bull';
|
|
||||||
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
|
|
||||||
|
|
||||||
type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;
|
|
||||||
|
|
||||||
@Injectable()
|
|
||||||
export class ProcessorService {
|
|
||||||
constructor(
|
|
||||||
private moduleRef: ModuleRef,
|
|
||||||
// TODO refactor to domain
|
|
||||||
private metadataProcessor: MetadataExtractionProcessor,
|
|
||||||
|
|
||||||
private facialRecognitionService: FacialRecognitionService,
|
|
||||||
private jobService: JobService,
|
|
||||||
private mediaService: MediaService,
|
|
||||||
private metadataService: MetadataService,
|
|
||||||
private personService: PersonService,
|
|
||||||
private searchService: SearchService,
|
|
||||||
private smartInfoService: SmartInfoService,
|
|
||||||
private storageTemplateService: StorageTemplateService,
|
|
||||||
private storageService: StorageService,
|
|
||||||
private systemConfigService: SystemConfigService,
|
|
||||||
private userService: UserService,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
private logger = new Logger(ProcessorService.name);
|
|
||||||
|
|
||||||
private handlers: Record<JobName, JobHandler> = {
|
|
||||||
[JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
|
|
||||||
[JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(),
|
|
||||||
[JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data),
|
|
||||||
[JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data),
|
|
||||||
[JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data),
|
|
||||||
[JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data),
|
|
||||||
[JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data),
|
|
||||||
[JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(),
|
|
||||||
[JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(),
|
|
||||||
[JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(),
|
|
||||||
[JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data),
|
|
||||||
[JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data),
|
|
||||||
[JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data),
|
|
||||||
[JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data),
|
|
||||||
[JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data),
|
|
||||||
[JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
|
|
||||||
[JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
|
|
||||||
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
|
|
||||||
[JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
|
|
||||||
[JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
|
|
||||||
[JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),
|
|
||||||
[JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data),
|
|
||||||
[JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data),
|
|
||||||
[JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),
|
|
||||||
[JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data),
|
|
||||||
[JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data),
|
|
||||||
[JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data),
|
|
||||||
[JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data),
|
|
||||||
[JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data),
|
|
||||||
[JobName.PERSON_CLEANUP]: () => this.personService.handlePersonCleanup(),
|
|
||||||
[JobName.QUEUE_SIDECAR]: (data) => this.metadataService.handleQueueSidecar(data),
|
|
||||||
[JobName.SIDECAR_DISCOVERY]: (data) => this.metadataService.handleSidecarDiscovery(data),
|
|
||||||
[JobName.SIDECAR_SYNC]: () => this.metadataService.handleSidecarSync(),
|
|
||||||
};
|
|
||||||
|
|
||||||
async init() {
|
|
||||||
const queueSeen: Partial<Record<QueueName, boolean>> = {};
|
|
||||||
|
|
||||||
for (const jobName of Object.values(JobName)) {
|
|
||||||
const handler = this.handlers[jobName];
|
|
||||||
const queueName = JOBS_TO_QUEUE[jobName];
|
|
||||||
const queue = this.moduleRef.get<Queue>(getQueueToken(queueName), { strict: false });
|
|
||||||
|
|
||||||
// only set concurrency on the first job for a queue, since concurrency stacks
|
|
||||||
const seen = queueSeen[queueName];
|
|
||||||
const concurrency = seen ? 0 : QUEUE_TO_CONCURRENCY[queueName];
|
|
||||||
queueSeen[queueName] = true;
|
|
||||||
|
|
||||||
await queue.isReady();
|
|
||||||
|
|
||||||
queue.process(jobName, concurrency, async (job): Promise<void> => {
|
|
||||||
try {
|
|
||||||
const success = await handler(job.data);
|
|
||||||
if (success) {
|
|
||||||
await this.jobService.onDone({ name: jobName, data: job.data } as JobItem);
|
|
||||||
}
|
|
||||||
} catch (error: Error | any) {
|
|
||||||
this.logger.error(`Unable to run job handler: ${error}`, error?.stack, job.data);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -0,0 +1,73 @@
|
|||||||
|
import { ApiProperty } from '@nestjs/swagger';
|
||||||
|
import { Type } from 'class-transformer';
|
||||||
|
import { IsInt, IsObject, IsPositive, ValidateNested } from 'class-validator';
|
||||||
|
import { QueueName } from '../../job';
|
||||||
|
|
||||||
|
export class JobSettingsDto {
|
||||||
|
@IsInt()
|
||||||
|
@IsPositive()
|
||||||
|
@ApiProperty({ type: 'integer' })
|
||||||
|
concurrency!: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.THUMBNAIL_GENERATION]!: JobSettingsDto;
|
||||||
|
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.METADATA_EXTRACTION]!: JobSettingsDto;
|
||||||
|
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.VIDEO_CONVERSION]!: JobSettingsDto;
|
||||||
|
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.OBJECT_TAGGING]!: JobSettingsDto;
|
||||||
|
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.CLIP_ENCODING]!: JobSettingsDto;
|
||||||
|
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto;
|
||||||
|
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.BACKGROUND_TASK]!: JobSettingsDto;
|
||||||
|
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.SEARCH]!: JobSettingsDto;
|
||||||
|
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.RECOGNIZE_FACES]!: JobSettingsDto;
|
||||||
|
|
||||||
|
@ApiProperty({ type: JobSettingsDto })
|
||||||
|
@ValidateNested()
|
||||||
|
@IsObject()
|
||||||
|
@Type(() => JobSettingsDto)
|
||||||
|
[QueueName.SIDECAR]!: JobSettingsDto;
|
||||||
|
}
|
||||||
@ -0,0 +1,103 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import {
|
||||||
|
notificationController,
|
||||||
|
NotificationType
|
||||||
|
} from '$lib/components/shared-components/notification/notification';
|
||||||
|
import { api, JobName, SystemConfigJobDto } from '@api';
|
||||||
|
import { isEqual } from 'lodash-es';
|
||||||
|
import { fade } from 'svelte/transition';
|
||||||
|
import { handleError } from '../../../../utils/handle-error';
|
||||||
|
import SettingButtonsRow from '../setting-buttons-row.svelte';
|
||||||
|
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
|
||||||
|
|
||||||
|
export let jobConfig: SystemConfigJobDto; // this is the config that is being edited
|
||||||
|
|
||||||
|
let savedConfig: SystemConfigJobDto;
|
||||||
|
let defaultConfig: SystemConfigJobDto;
|
||||||
|
|
||||||
|
const ignoredJobs = [JobName.BackgroundTask, JobName.Search] as JobName[];
|
||||||
|
const jobNames = Object.values(JobName).filter(
|
||||||
|
(jobName) => !ignoredJobs.includes(jobName as JobName)
|
||||||
|
);
|
||||||
|
|
||||||
|
async function getConfigs() {
|
||||||
|
[savedConfig, defaultConfig] = await Promise.all([
|
||||||
|
api.systemConfigApi.getConfig().then((res) => res.data.job),
|
||||||
|
api.systemConfigApi.getDefaults().then((res) => res.data.job)
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveSetting() {
|
||||||
|
try {
|
||||||
|
const { data: configs } = await api.systemConfigApi.getConfig();
|
||||||
|
|
||||||
|
const result = await api.systemConfigApi.updateConfig({
|
||||||
|
systemConfigDto: {
|
||||||
|
...configs,
|
||||||
|
job: jobConfig
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
jobConfig = { ...result.data.job };
|
||||||
|
savedConfig = { ...result.data.job };
|
||||||
|
|
||||||
|
notificationController.show({ message: 'Job settings saved', type: NotificationType.Info });
|
||||||
|
} catch (error) {
|
||||||
|
handleError(error, 'Unable to save settings');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function reset() {
|
||||||
|
const { data: resetConfig } = await api.systemConfigApi.getConfig();
|
||||||
|
|
||||||
|
jobConfig = { ...resetConfig.job };
|
||||||
|
savedConfig = { ...resetConfig.job };
|
||||||
|
|
||||||
|
notificationController.show({
|
||||||
|
message: 'Reset Job settings to the recent saved settings',
|
||||||
|
type: NotificationType.Info
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resetToDefault() {
|
||||||
|
const { data: configs } = await api.systemConfigApi.getDefaults();
|
||||||
|
|
||||||
|
jobConfig = { ...configs.job };
|
||||||
|
defaultConfig = { ...configs.job };
|
||||||
|
|
||||||
|
notificationController.show({
|
||||||
|
message: 'Reset Job settings to default',
|
||||||
|
type: NotificationType.Info
|
||||||
|
});
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
{#await getConfigs() then}
|
||||||
|
<div in:fade={{ duration: 500 }}>
|
||||||
|
<form autocomplete="off" on:submit|preventDefault>
|
||||||
|
{#each jobNames as jobName}
|
||||||
|
<div class="flex flex-col gap-4 ml-4 mt-4">
|
||||||
|
<SettingInputField
|
||||||
|
inputType={SettingInputFieldType.NUMBER}
|
||||||
|
label="{api.getJobName(jobName)} Concurrency"
|
||||||
|
desc=""
|
||||||
|
bind:value={jobConfig[jobName].concurrency}
|
||||||
|
required={true}
|
||||||
|
isEdited={!(jobConfig[jobName].concurrency == savedConfig[jobName].concurrency)}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
|
||||||
|
<div class="ml-4">
|
||||||
|
<SettingButtonsRow
|
||||||
|
on:reset={reset}
|
||||||
|
on:save={saveSetting}
|
||||||
|
on:reset-to-default={resetToDefault}
|
||||||
|
showResetToDefault={!isEqual(savedConfig, defaultConfig)}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
{/await}
|
||||||
|
</div>
|
||||||
Loading…
Reference in New Issue