mirror of https://github.com/immich-app/immich.git
feat(server): dynamic job concurrency (#2622)
* feat(server): dynamic job concurrency * styling and add setting info to top of the job list * regenerate api * remove DETECT_OBJECT job --------- Co-authored-by: Alex Tran <alex.tran1502@gmail.com>pull/2634/head
Jason Rasmussen
GitHub
parent
656dc08406
commit
2493dfaba3
@ -0,0 +1,15 @@
|
||||
# openapi.model.JobSettingsDto
|
||||
|
||||
## Load the model package
|
||||
```dart
|
||||
import 'package:openapi/api.dart';
|
||||
```
|
||||
|
||||
## Properties
|
||||
Name | Type | Description | Notes
|
||||
------------ | ------------- | ------------- | -------------
|
||||
**concurrency** | **int** | |
|
||||
|
||||
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||
|
||||
|
||||
@ -0,0 +1,24 @@
|
||||
# openapi.model.SystemConfigJobDto
|
||||
|
||||
## Load the model package
|
||||
```dart
|
||||
import 'package:openapi/api.dart';
|
||||
```
|
||||
|
||||
## Properties
|
||||
Name | Type | Description | Notes
|
||||
------------ | ------------- | ------------- | -------------
|
||||
**thumbnailGeneration** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
**metadataExtraction** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
**videoConversion** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
**objectTagging** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
**clipEncoding** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
**storageTemplateMigration** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
**backgroundTask** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
**search** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
**recognizeFaces** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
**sidecar** | [**JobSettingsDto**](JobSettingsDto.md) | |
|
||||
|
||||
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||
|
||||
|
||||
@ -0,0 +1,109 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
class JobSettingsDto {
|
||||
/// Returns a new [JobSettingsDto] instance.
|
||||
JobSettingsDto({
|
||||
required this.concurrency,
|
||||
});
|
||||
|
||||
int concurrency;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) => identical(this, other) || other is JobSettingsDto &&
|
||||
other.concurrency == concurrency;
|
||||
|
||||
@override
|
||||
int get hashCode =>
|
||||
// ignore: unnecessary_parenthesis
|
||||
(concurrency.hashCode);
|
||||
|
||||
@override
|
||||
String toString() => 'JobSettingsDto[concurrency=$concurrency]';
|
||||
|
||||
Map<String, dynamic> toJson() {
|
||||
final json = <String, dynamic>{};
|
||||
json[r'concurrency'] = this.concurrency;
|
||||
return json;
|
||||
}
|
||||
|
||||
/// Returns a new [JobSettingsDto] instance and imports its values from
|
||||
/// [value] if it's a [Map], null otherwise.
|
||||
// ignore: prefer_constructors_over_static_methods
|
||||
static JobSettingsDto? fromJson(dynamic value) {
|
||||
if (value is Map) {
|
||||
final json = value.cast<String, dynamic>();
|
||||
|
||||
// Ensure that the map contains the required keys.
|
||||
// Note 1: the values aren't checked for validity beyond being non-null.
|
||||
// Note 2: this code is stripped in release mode!
|
||||
assert(() {
|
||||
requiredKeys.forEach((key) {
|
||||
assert(json.containsKey(key), 'Required key "JobSettingsDto[$key]" is missing from JSON.');
|
||||
assert(json[key] != null, 'Required key "JobSettingsDto[$key]" has a null value in JSON.');
|
||||
});
|
||||
return true;
|
||||
}());
|
||||
|
||||
return JobSettingsDto(
|
||||
concurrency: mapValueOfType<int>(json, r'concurrency')!,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static List<JobSettingsDto> listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <JobSettingsDto>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = JobSettingsDto.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
|
||||
static Map<String, JobSettingsDto> mapFromJson(dynamic json) {
|
||||
final map = <String, JobSettingsDto>{};
|
||||
if (json is Map && json.isNotEmpty) {
|
||||
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
|
||||
for (final entry in json.entries) {
|
||||
final value = JobSettingsDto.fromJson(entry.value);
|
||||
if (value != null) {
|
||||
map[entry.key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// maps a json object with a list of JobSettingsDto-objects as value to a dart map
|
||||
static Map<String, List<JobSettingsDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
|
||||
final map = <String, List<JobSettingsDto>>{};
|
||||
if (json is Map && json.isNotEmpty) {
|
||||
// ignore: parameter_assignments
|
||||
json = json.cast<String, dynamic>();
|
||||
for (final entry in json.entries) {
|
||||
map[entry.key] = JobSettingsDto.listFromJson(entry.value, growable: growable,);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/// The list of required keys that must be present in a JSON.
|
||||
static const requiredKeys = <String>{
|
||||
'concurrency',
|
||||
};
|
||||
}
|
||||
|
||||
@ -0,0 +1,181 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
class SystemConfigJobDto {
|
||||
/// Returns a new [SystemConfigJobDto] instance.
|
||||
SystemConfigJobDto({
|
||||
required this.thumbnailGeneration,
|
||||
required this.metadataExtraction,
|
||||
required this.videoConversion,
|
||||
required this.objectTagging,
|
||||
required this.clipEncoding,
|
||||
required this.storageTemplateMigration,
|
||||
required this.backgroundTask,
|
||||
required this.search,
|
||||
required this.recognizeFaces,
|
||||
required this.sidecar,
|
||||
});
|
||||
|
||||
JobSettingsDto thumbnailGeneration;
|
||||
|
||||
JobSettingsDto metadataExtraction;
|
||||
|
||||
JobSettingsDto videoConversion;
|
||||
|
||||
JobSettingsDto objectTagging;
|
||||
|
||||
JobSettingsDto clipEncoding;
|
||||
|
||||
JobSettingsDto storageTemplateMigration;
|
||||
|
||||
JobSettingsDto backgroundTask;
|
||||
|
||||
JobSettingsDto search;
|
||||
|
||||
JobSettingsDto recognizeFaces;
|
||||
|
||||
JobSettingsDto sidecar;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) => identical(this, other) || other is SystemConfigJobDto &&
|
||||
other.thumbnailGeneration == thumbnailGeneration &&
|
||||
other.metadataExtraction == metadataExtraction &&
|
||||
other.videoConversion == videoConversion &&
|
||||
other.objectTagging == objectTagging &&
|
||||
other.clipEncoding == clipEncoding &&
|
||||
other.storageTemplateMigration == storageTemplateMigration &&
|
||||
other.backgroundTask == backgroundTask &&
|
||||
other.search == search &&
|
||||
other.recognizeFaces == recognizeFaces &&
|
||||
other.sidecar == sidecar;
|
||||
|
||||
@override
|
||||
int get hashCode =>
|
||||
// ignore: unnecessary_parenthesis
|
||||
(thumbnailGeneration.hashCode) +
|
||||
(metadataExtraction.hashCode) +
|
||||
(videoConversion.hashCode) +
|
||||
(objectTagging.hashCode) +
|
||||
(clipEncoding.hashCode) +
|
||||
(storageTemplateMigration.hashCode) +
|
||||
(backgroundTask.hashCode) +
|
||||
(search.hashCode) +
|
||||
(recognizeFaces.hashCode) +
|
||||
(sidecar.hashCode);
|
||||
|
||||
@override
|
||||
String toString() => 'SystemConfigJobDto[thumbnailGeneration=$thumbnailGeneration, metadataExtraction=$metadataExtraction, videoConversion=$videoConversion, objectTagging=$objectTagging, clipEncoding=$clipEncoding, storageTemplateMigration=$storageTemplateMigration, backgroundTask=$backgroundTask, search=$search, recognizeFaces=$recognizeFaces, sidecar=$sidecar]';
|
||||
|
||||
Map<String, dynamic> toJson() {
|
||||
final json = <String, dynamic>{};
|
||||
json[r'thumbnailGeneration'] = this.thumbnailGeneration;
|
||||
json[r'metadataExtraction'] = this.metadataExtraction;
|
||||
json[r'videoConversion'] = this.videoConversion;
|
||||
json[r'objectTagging'] = this.objectTagging;
|
||||
json[r'clipEncoding'] = this.clipEncoding;
|
||||
json[r'storageTemplateMigration'] = this.storageTemplateMigration;
|
||||
json[r'backgroundTask'] = this.backgroundTask;
|
||||
json[r'search'] = this.search;
|
||||
json[r'recognizeFaces'] = this.recognizeFaces;
|
||||
json[r'sidecar'] = this.sidecar;
|
||||
return json;
|
||||
}
|
||||
|
||||
/// Returns a new [SystemConfigJobDto] instance and imports its values from
|
||||
/// [value] if it's a [Map], null otherwise.
|
||||
// ignore: prefer_constructors_over_static_methods
|
||||
static SystemConfigJobDto? fromJson(dynamic value) {
|
||||
if (value is Map) {
|
||||
final json = value.cast<String, dynamic>();
|
||||
|
||||
// Ensure that the map contains the required keys.
|
||||
// Note 1: the values aren't checked for validity beyond being non-null.
|
||||
// Note 2: this code is stripped in release mode!
|
||||
assert(() {
|
||||
requiredKeys.forEach((key) {
|
||||
assert(json.containsKey(key), 'Required key "SystemConfigJobDto[$key]" is missing from JSON.');
|
||||
assert(json[key] != null, 'Required key "SystemConfigJobDto[$key]" has a null value in JSON.');
|
||||
});
|
||||
return true;
|
||||
}());
|
||||
|
||||
return SystemConfigJobDto(
|
||||
thumbnailGeneration: JobSettingsDto.fromJson(json[r'thumbnailGeneration'])!,
|
||||
metadataExtraction: JobSettingsDto.fromJson(json[r'metadataExtraction'])!,
|
||||
videoConversion: JobSettingsDto.fromJson(json[r'videoConversion'])!,
|
||||
objectTagging: JobSettingsDto.fromJson(json[r'objectTagging'])!,
|
||||
clipEncoding: JobSettingsDto.fromJson(json[r'clipEncoding'])!,
|
||||
storageTemplateMigration: JobSettingsDto.fromJson(json[r'storageTemplateMigration'])!,
|
||||
backgroundTask: JobSettingsDto.fromJson(json[r'backgroundTask'])!,
|
||||
search: JobSettingsDto.fromJson(json[r'search'])!,
|
||||
recognizeFaces: JobSettingsDto.fromJson(json[r'recognizeFaces'])!,
|
||||
sidecar: JobSettingsDto.fromJson(json[r'sidecar'])!,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static List<SystemConfigJobDto> listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <SystemConfigJobDto>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = SystemConfigJobDto.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
|
||||
static Map<String, SystemConfigJobDto> mapFromJson(dynamic json) {
|
||||
final map = <String, SystemConfigJobDto>{};
|
||||
if (json is Map && json.isNotEmpty) {
|
||||
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
|
||||
for (final entry in json.entries) {
|
||||
final value = SystemConfigJobDto.fromJson(entry.value);
|
||||
if (value != null) {
|
||||
map[entry.key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// maps a json object with a list of SystemConfigJobDto-objects as value to a dart map
|
||||
static Map<String, List<SystemConfigJobDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
|
||||
final map = <String, List<SystemConfigJobDto>>{};
|
||||
if (json is Map && json.isNotEmpty) {
|
||||
// ignore: parameter_assignments
|
||||
json = json.cast<String, dynamic>();
|
||||
for (final entry in json.entries) {
|
||||
map[entry.key] = SystemConfigJobDto.listFromJson(entry.value, growable: growable,);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/// The list of required keys that must be present in a JSON.
|
||||
static const requiredKeys = <String>{
|
||||
'thumbnailGeneration',
|
||||
'metadataExtraction',
|
||||
'videoConversion',
|
||||
'objectTagging',
|
||||
'clipEncoding',
|
||||
'storageTemplateMigration',
|
||||
'backgroundTask',
|
||||
'search',
|
||||
'recognizeFaces',
|
||||
'sidecar',
|
||||
};
|
||||
}
|
||||
|
||||
@ -0,0 +1,27 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
import 'package:openapi/api.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
// tests for JobSettingsDto
|
||||
void main() {
|
||||
// final instance = JobSettingsDto();
|
||||
|
||||
group('test JobSettingsDto', () {
|
||||
// int concurrency
|
||||
test('to test the property `concurrency`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
@ -0,0 +1,72 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
import 'package:openapi/api.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
// tests for SystemConfigJobDto
|
||||
void main() {
|
||||
// final instance = SystemConfigJobDto();
|
||||
|
||||
group('test SystemConfigJobDto', () {
|
||||
// JobSettingsDto thumbnailGeneration
|
||||
test('to test the property `thumbnailGeneration`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// JobSettingsDto metadataExtraction
|
||||
test('to test the property `metadataExtraction`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// JobSettingsDto videoConversion
|
||||
test('to test the property `videoConversion`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// JobSettingsDto objectTagging
|
||||
test('to test the property `objectTagging`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// JobSettingsDto clipEncoding
|
||||
test('to test the property `clipEncoding`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// JobSettingsDto storageTemplateMigration
|
||||
test('to test the property `storageTemplateMigration`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// JobSettingsDto backgroundTask
|
||||
test('to test the property `backgroundTask`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// JobSettingsDto search
|
||||
test('to test the property `search`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// JobSettingsDto recognizeFaces
|
||||
test('to test the property `recognizeFaces`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// JobSettingsDto sidecar
|
||||
test('to test the property `sidecar`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
@ -0,0 +1,75 @@
|
||||
import {
|
||||
FacialRecognitionService,
|
||||
IDeleteFilesJob,
|
||||
JobName,
|
||||
JobService,
|
||||
MediaService,
|
||||
MetadataService,
|
||||
PersonService,
|
||||
SearchService,
|
||||
SmartInfoService,
|
||||
StorageService,
|
||||
StorageTemplateService,
|
||||
SystemConfigService,
|
||||
UserService,
|
||||
} from '@app/domain';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
|
||||
|
||||
@Injectable()
|
||||
export class AppService {
|
||||
constructor(
|
||||
// TODO refactor to domain
|
||||
private metadataProcessor: MetadataExtractionProcessor,
|
||||
|
||||
private facialRecognitionService: FacialRecognitionService,
|
||||
private jobService: JobService,
|
||||
private mediaService: MediaService,
|
||||
private metadataService: MetadataService,
|
||||
private personService: PersonService,
|
||||
private searchService: SearchService,
|
||||
private smartInfoService: SmartInfoService,
|
||||
private storageTemplateService: StorageTemplateService,
|
||||
private storageService: StorageService,
|
||||
private systemConfigService: SystemConfigService,
|
||||
private userService: UserService,
|
||||
) {}
|
||||
|
||||
async init() {
|
||||
await this.jobService.registerHandlers({
|
||||
[JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
|
||||
[JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(),
|
||||
[JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data),
|
||||
[JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data),
|
||||
[JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data),
|
||||
[JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data),
|
||||
[JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data),
|
||||
[JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(),
|
||||
[JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(),
|
||||
[JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(),
|
||||
[JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data),
|
||||
[JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data),
|
||||
[JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data),
|
||||
[JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data),
|
||||
[JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data),
|
||||
[JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
|
||||
[JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
|
||||
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
|
||||
[JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
|
||||
[JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
|
||||
[JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),
|
||||
[JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data),
|
||||
[JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data),
|
||||
[JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),
|
||||
[JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data),
|
||||
[JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data),
|
||||
[JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data),
|
||||
[JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data),
|
||||
[JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data),
|
||||
[JobName.PERSON_CLEANUP]: () => this.personService.handlePersonCleanup(),
|
||||
[JobName.QUEUE_SIDECAR]: (data) => this.metadataService.handleQueueSidecar(data),
|
||||
[JobName.SIDECAR_DISCOVERY]: (data) => this.metadataService.handleSidecarDiscovery(data),
|
||||
[JobName.SIDECAR_SYNC]: () => this.metadataService.handleSidecarSync(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -1,113 +0,0 @@
|
||||
import {
|
||||
FacialRecognitionService,
|
||||
IDeleteFilesJob,
|
||||
JobItem,
|
||||
JobName,
|
||||
JobService,
|
||||
JOBS_TO_QUEUE,
|
||||
MediaService,
|
||||
MetadataService,
|
||||
PersonService,
|
||||
QueueName,
|
||||
QUEUE_TO_CONCURRENCY,
|
||||
SearchService,
|
||||
SmartInfoService,
|
||||
StorageService,
|
||||
StorageTemplateService,
|
||||
SystemConfigService,
|
||||
UserService,
|
||||
} from '@app/domain';
|
||||
import { getQueueToken } from '@nestjs/bull';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
import { Queue } from 'bull';
|
||||
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
|
||||
|
||||
type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;
|
||||
|
||||
@Injectable()
|
||||
export class ProcessorService {
|
||||
constructor(
|
||||
private moduleRef: ModuleRef,
|
||||
// TODO refactor to domain
|
||||
private metadataProcessor: MetadataExtractionProcessor,
|
||||
|
||||
private facialRecognitionService: FacialRecognitionService,
|
||||
private jobService: JobService,
|
||||
private mediaService: MediaService,
|
||||
private metadataService: MetadataService,
|
||||
private personService: PersonService,
|
||||
private searchService: SearchService,
|
||||
private smartInfoService: SmartInfoService,
|
||||
private storageTemplateService: StorageTemplateService,
|
||||
private storageService: StorageService,
|
||||
private systemConfigService: SystemConfigService,
|
||||
private userService: UserService,
|
||||
) {}
|
||||
|
||||
private logger = new Logger(ProcessorService.name);
|
||||
|
||||
private handlers: Record<JobName, JobHandler> = {
|
||||
[JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
|
||||
[JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(),
|
||||
[JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data),
|
||||
[JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data),
|
||||
[JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data),
|
||||
[JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data),
|
||||
[JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data),
|
||||
[JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(),
|
||||
[JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(),
|
||||
[JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(),
|
||||
[JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data),
|
||||
[JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data),
|
||||
[JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data),
|
||||
[JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data),
|
||||
[JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data),
|
||||
[JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
|
||||
[JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
|
||||
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
|
||||
[JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
|
||||
[JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
|
||||
[JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),
|
||||
[JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data),
|
||||
[JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data),
|
||||
[JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),
|
||||
[JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data),
|
||||
[JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data),
|
||||
[JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data),
|
||||
[JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data),
|
||||
[JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data),
|
||||
[JobName.PERSON_CLEANUP]: () => this.personService.handlePersonCleanup(),
|
||||
[JobName.QUEUE_SIDECAR]: (data) => this.metadataService.handleQueueSidecar(data),
|
||||
[JobName.SIDECAR_DISCOVERY]: (data) => this.metadataService.handleSidecarDiscovery(data),
|
||||
[JobName.SIDECAR_SYNC]: () => this.metadataService.handleSidecarSync(),
|
||||
};
|
||||
|
||||
async init() {
|
||||
const queueSeen: Partial<Record<QueueName, boolean>> = {};
|
||||
|
||||
for (const jobName of Object.values(JobName)) {
|
||||
const handler = this.handlers[jobName];
|
||||
const queueName = JOBS_TO_QUEUE[jobName];
|
||||
const queue = this.moduleRef.get<Queue>(getQueueToken(queueName), { strict: false });
|
||||
|
||||
// only set concurrency on the first job for a queue, since concurrency stacks
|
||||
const seen = queueSeen[queueName];
|
||||
const concurrency = seen ? 0 : QUEUE_TO_CONCURRENCY[queueName];
|
||||
queueSeen[queueName] = true;
|
||||
|
||||
await queue.isReady();
|
||||
|
||||
queue.process(jobName, concurrency, async (job): Promise<void> => {
|
||||
try {
|
||||
const success = await handler(job.data);
|
||||
if (success) {
|
||||
await this.jobService.onDone({ name: jobName, data: job.data } as JobItem);
|
||||
}
|
||||
} catch (error: Error | any) {
|
||||
this.logger.error(`Unable to run job handler: ${error}`, error?.stack, job.data);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,73 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { Type } from 'class-transformer';
|
||||
import { IsInt, IsObject, IsPositive, ValidateNested } from 'class-validator';
|
||||
import { QueueName } from '../../job';
|
||||
|
||||
export class JobSettingsDto {
|
||||
@IsInt()
|
||||
@IsPositive()
|
||||
@ApiProperty({ type: 'integer' })
|
||||
concurrency!: number;
|
||||
}
|
||||
|
||||
export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.THUMBNAIL_GENERATION]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.METADATA_EXTRACTION]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.VIDEO_CONVERSION]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.OBJECT_TAGGING]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.CLIP_ENCODING]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.BACKGROUND_TASK]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.SEARCH]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.RECOGNIZE_FACES]!: JobSettingsDto;
|
||||
|
||||
@ApiProperty({ type: JobSettingsDto })
|
||||
@ValidateNested()
|
||||
@IsObject()
|
||||
@Type(() => JobSettingsDto)
|
||||
[QueueName.SIDECAR]!: JobSettingsDto;
|
||||
}
|
||||
@ -0,0 +1,103 @@
|
||||
<script lang="ts">
|
||||
import {
|
||||
notificationController,
|
||||
NotificationType
|
||||
} from '$lib/components/shared-components/notification/notification';
|
||||
import { api, JobName, SystemConfigJobDto } from '@api';
|
||||
import { isEqual } from 'lodash-es';
|
||||
import { fade } from 'svelte/transition';
|
||||
import { handleError } from '../../../../utils/handle-error';
|
||||
import SettingButtonsRow from '../setting-buttons-row.svelte';
|
||||
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
|
||||
|
||||
export let jobConfig: SystemConfigJobDto; // this is the config that is being edited
|
||||
|
||||
let savedConfig: SystemConfigJobDto;
|
||||
let defaultConfig: SystemConfigJobDto;
|
||||
|
||||
const ignoredJobs = [JobName.BackgroundTask, JobName.Search] as JobName[];
|
||||
const jobNames = Object.values(JobName).filter(
|
||||
(jobName) => !ignoredJobs.includes(jobName as JobName)
|
||||
);
|
||||
|
||||
async function getConfigs() {
|
||||
[savedConfig, defaultConfig] = await Promise.all([
|
||||
api.systemConfigApi.getConfig().then((res) => res.data.job),
|
||||
api.systemConfigApi.getDefaults().then((res) => res.data.job)
|
||||
]);
|
||||
}
|
||||
|
||||
async function saveSetting() {
|
||||
try {
|
||||
const { data: configs } = await api.systemConfigApi.getConfig();
|
||||
|
||||
const result = await api.systemConfigApi.updateConfig({
|
||||
systemConfigDto: {
|
||||
...configs,
|
||||
job: jobConfig
|
||||
}
|
||||
});
|
||||
|
||||
jobConfig = { ...result.data.job };
|
||||
savedConfig = { ...result.data.job };
|
||||
|
||||
notificationController.show({ message: 'Job settings saved', type: NotificationType.Info });
|
||||
} catch (error) {
|
||||
handleError(error, 'Unable to save settings');
|
||||
}
|
||||
}
|
||||
|
||||
async function reset() {
|
||||
const { data: resetConfig } = await api.systemConfigApi.getConfig();
|
||||
|
||||
jobConfig = { ...resetConfig.job };
|
||||
savedConfig = { ...resetConfig.job };
|
||||
|
||||
notificationController.show({
|
||||
message: 'Reset Job settings to the recent saved settings',
|
||||
type: NotificationType.Info
|
||||
});
|
||||
}
|
||||
|
||||
async function resetToDefault() {
|
||||
const { data: configs } = await api.systemConfigApi.getDefaults();
|
||||
|
||||
jobConfig = { ...configs.job };
|
||||
defaultConfig = { ...configs.job };
|
||||
|
||||
notificationController.show({
|
||||
message: 'Reset Job settings to default',
|
||||
type: NotificationType.Info
|
||||
});
|
||||
}
|
||||
</script>
|
||||
|
||||
<div>
|
||||
{#await getConfigs() then}
|
||||
<div in:fade={{ duration: 500 }}>
|
||||
<form autocomplete="off" on:submit|preventDefault>
|
||||
{#each jobNames as jobName}
|
||||
<div class="flex flex-col gap-4 ml-4 mt-4">
|
||||
<SettingInputField
|
||||
inputType={SettingInputFieldType.NUMBER}
|
||||
label="{api.getJobName(jobName)} Concurrency"
|
||||
desc=""
|
||||
bind:value={jobConfig[jobName].concurrency}
|
||||
required={true}
|
||||
isEdited={!(jobConfig[jobName].concurrency == savedConfig[jobName].concurrency)}
|
||||
/>
|
||||
</div>
|
||||
{/each}
|
||||
|
||||
<div class="ml-4">
|
||||
<SettingButtonsRow
|
||||
on:reset={reset}
|
||||
on:save={saveSetting}
|
||||
on:reset-to-default={resetToDefault}
|
||||
showResetToDefault={!isEqual(savedConfig, defaultConfig)}
|
||||
/>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
{/await}
|
||||
</div>
|
||||
Loading…
Reference in New Issue