mirror of https://github.com/immich-app/immich.git
refactor: job vs queue naming (#23902)
Jason Rasmussen
GitHub
parent
1200bfad13
commit
d784d431d0
@ -1,94 +0,0 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.18
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
|
||||
class JobCommand {
|
||||
/// Instantiate a new enum with the provided [value].
|
||||
const JobCommand._(this.value);
|
||||
|
||||
/// The underlying value of this enum member.
|
||||
final String value;
|
||||
|
||||
@override
|
||||
String toString() => value;
|
||||
|
||||
String toJson() => value;
|
||||
|
||||
static const start = JobCommand._(r'start');
|
||||
static const pause = JobCommand._(r'pause');
|
||||
static const resume = JobCommand._(r'resume');
|
||||
static const empty = JobCommand._(r'empty');
|
||||
static const clearFailed = JobCommand._(r'clear-failed');
|
||||
|
||||
/// List of all possible values in this [enum][JobCommand].
|
||||
static const values = <JobCommand>[
|
||||
start,
|
||||
pause,
|
||||
resume,
|
||||
empty,
|
||||
clearFailed,
|
||||
];
|
||||
|
||||
static JobCommand? fromJson(dynamic value) => JobCommandTypeTransformer().decode(value);
|
||||
|
||||
static List<JobCommand> listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <JobCommand>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = JobCommand.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
}
|
||||
|
||||
/// Transformation class that can [encode] an instance of [JobCommand] to String,
|
||||
/// and [decode] dynamic data back to [JobCommand].
|
||||
class JobCommandTypeTransformer {
|
||||
factory JobCommandTypeTransformer() => _instance ??= const JobCommandTypeTransformer._();
|
||||
|
||||
const JobCommandTypeTransformer._();
|
||||
|
||||
String encode(JobCommand data) => data.value;
|
||||
|
||||
/// Decodes a [dynamic value][data] to a JobCommand.
|
||||
///
|
||||
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
|
||||
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
|
||||
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
|
||||
///
|
||||
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
|
||||
/// and users are still using an old app with the old code.
|
||||
JobCommand? decode(dynamic data, {bool allowNull = true}) {
|
||||
if (data != null) {
|
||||
switch (data) {
|
||||
case r'start': return JobCommand.start;
|
||||
case r'pause': return JobCommand.pause;
|
||||
case r'resume': return JobCommand.resume;
|
||||
case r'empty': return JobCommand.empty;
|
||||
case r'clear-failed': return JobCommand.clearFailed;
|
||||
default:
|
||||
if (!allowNull) {
|
||||
throw ArgumentError('Unknown enum value to decode: $data');
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Singleton [JobCommandTypeTransformer] instance.
|
||||
static JobCommandTypeTransformer? _instance;
|
||||
}
|
||||
|
||||
@ -1,127 +0,0 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.18
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
|
||||
class JobName {
|
||||
/// Instantiate a new enum with the provided [value].
|
||||
const JobName._(this.value);
|
||||
|
||||
/// The underlying value of this enum member.
|
||||
final String value;
|
||||
|
||||
@override
|
||||
String toString() => value;
|
||||
|
||||
String toJson() => value;
|
||||
|
||||
static const thumbnailGeneration = JobName._(r'thumbnailGeneration');
|
||||
static const metadataExtraction = JobName._(r'metadataExtraction');
|
||||
static const videoConversion = JobName._(r'videoConversion');
|
||||
static const faceDetection = JobName._(r'faceDetection');
|
||||
static const facialRecognition = JobName._(r'facialRecognition');
|
||||
static const smartSearch = JobName._(r'smartSearch');
|
||||
static const duplicateDetection = JobName._(r'duplicateDetection');
|
||||
static const backgroundTask = JobName._(r'backgroundTask');
|
||||
static const storageTemplateMigration = JobName._(r'storageTemplateMigration');
|
||||
static const migration = JobName._(r'migration');
|
||||
static const search = JobName._(r'search');
|
||||
static const sidecar = JobName._(r'sidecar');
|
||||
static const library_ = JobName._(r'library');
|
||||
static const notifications = JobName._(r'notifications');
|
||||
static const backupDatabase = JobName._(r'backupDatabase');
|
||||
static const ocr = JobName._(r'ocr');
|
||||
|
||||
/// List of all possible values in this [enum][JobName].
|
||||
static const values = <JobName>[
|
||||
thumbnailGeneration,
|
||||
metadataExtraction,
|
||||
videoConversion,
|
||||
faceDetection,
|
||||
facialRecognition,
|
||||
smartSearch,
|
||||
duplicateDetection,
|
||||
backgroundTask,
|
||||
storageTemplateMigration,
|
||||
migration,
|
||||
search,
|
||||
sidecar,
|
||||
library_,
|
||||
notifications,
|
||||
backupDatabase,
|
||||
ocr,
|
||||
];
|
||||
|
||||
static JobName? fromJson(dynamic value) => JobNameTypeTransformer().decode(value);
|
||||
|
||||
static List<JobName> listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <JobName>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = JobName.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
}
|
||||
|
||||
/// Transformation class that can [encode] an instance of [JobName] to String,
|
||||
/// and [decode] dynamic data back to [JobName].
|
||||
class JobNameTypeTransformer {
|
||||
factory JobNameTypeTransformer() => _instance ??= const JobNameTypeTransformer._();
|
||||
|
||||
const JobNameTypeTransformer._();
|
||||
|
||||
String encode(JobName data) => data.value;
|
||||
|
||||
/// Decodes a [dynamic value][data] to a JobName.
|
||||
///
|
||||
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
|
||||
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
|
||||
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
|
||||
///
|
||||
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
|
||||
/// and users are still using an old app with the old code.
|
||||
JobName? decode(dynamic data, {bool allowNull = true}) {
|
||||
if (data != null) {
|
||||
switch (data) {
|
||||
case r'thumbnailGeneration': return JobName.thumbnailGeneration;
|
||||
case r'metadataExtraction': return JobName.metadataExtraction;
|
||||
case r'videoConversion': return JobName.videoConversion;
|
||||
case r'faceDetection': return JobName.faceDetection;
|
||||
case r'facialRecognition': return JobName.facialRecognition;
|
||||
case r'smartSearch': return JobName.smartSearch;
|
||||
case r'duplicateDetection': return JobName.duplicateDetection;
|
||||
case r'backgroundTask': return JobName.backgroundTask;
|
||||
case r'storageTemplateMigration': return JobName.storageTemplateMigration;
|
||||
case r'migration': return JobName.migration;
|
||||
case r'search': return JobName.search;
|
||||
case r'sidecar': return JobName.sidecar;
|
||||
case r'library': return JobName.library_;
|
||||
case r'notifications': return JobName.notifications;
|
||||
case r'backupDatabase': return JobName.backupDatabase;
|
||||
case r'ocr': return JobName.ocr;
|
||||
default:
|
||||
if (!allowNull) {
|
||||
throw ArgumentError('Unknown enum value to decode: $data');
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Singleton [JobNameTypeTransformer] instance.
|
||||
static JobNameTypeTransformer? _instance;
|
||||
}
|
||||
|
||||
@ -0,0 +1,94 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.18
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
|
||||
class QueueCommand {
|
||||
/// Instantiate a new enum with the provided [value].
|
||||
const QueueCommand._(this.value);
|
||||
|
||||
/// The underlying value of this enum member.
|
||||
final String value;
|
||||
|
||||
@override
|
||||
String toString() => value;
|
||||
|
||||
String toJson() => value;
|
||||
|
||||
static const start = QueueCommand._(r'start');
|
||||
static const pause = QueueCommand._(r'pause');
|
||||
static const resume = QueueCommand._(r'resume');
|
||||
static const empty = QueueCommand._(r'empty');
|
||||
static const clearFailed = QueueCommand._(r'clear-failed');
|
||||
|
||||
/// List of all possible values in this [enum][QueueCommand].
|
||||
static const values = <QueueCommand>[
|
||||
start,
|
||||
pause,
|
||||
resume,
|
||||
empty,
|
||||
clearFailed,
|
||||
];
|
||||
|
||||
static QueueCommand? fromJson(dynamic value) => QueueCommandTypeTransformer().decode(value);
|
||||
|
||||
static List<QueueCommand> listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <QueueCommand>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = QueueCommand.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
}
|
||||
|
||||
/// Transformation class that can [encode] an instance of [QueueCommand] to String,
|
||||
/// and [decode] dynamic data back to [QueueCommand].
|
||||
class QueueCommandTypeTransformer {
|
||||
factory QueueCommandTypeTransformer() => _instance ??= const QueueCommandTypeTransformer._();
|
||||
|
||||
const QueueCommandTypeTransformer._();
|
||||
|
||||
String encode(QueueCommand data) => data.value;
|
||||
|
||||
/// Decodes a [dynamic value][data] to a QueueCommand.
|
||||
///
|
||||
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
|
||||
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
|
||||
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
|
||||
///
|
||||
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
|
||||
/// and users are still using an old app with the old code.
|
||||
QueueCommand? decode(dynamic data, {bool allowNull = true}) {
|
||||
if (data != null) {
|
||||
switch (data) {
|
||||
case r'start': return QueueCommand.start;
|
||||
case r'pause': return QueueCommand.pause;
|
||||
case r'resume': return QueueCommand.resume;
|
||||
case r'empty': return QueueCommand.empty;
|
||||
case r'clear-failed': return QueueCommand.clearFailed;
|
||||
default:
|
||||
if (!allowNull) {
|
||||
throw ArgumentError('Unknown enum value to decode: $data');
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Singleton [QueueCommandTypeTransformer] instance.
|
||||
static QueueCommandTypeTransformer? _instance;
|
||||
}
|
||||
|
||||
@ -0,0 +1,127 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.18
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
|
||||
class QueueName {
|
||||
/// Instantiate a new enum with the provided [value].
|
||||
const QueueName._(this.value);
|
||||
|
||||
/// The underlying value of this enum member.
|
||||
final String value;
|
||||
|
||||
@override
|
||||
String toString() => value;
|
||||
|
||||
String toJson() => value;
|
||||
|
||||
static const thumbnailGeneration = QueueName._(r'thumbnailGeneration');
|
||||
static const metadataExtraction = QueueName._(r'metadataExtraction');
|
||||
static const videoConversion = QueueName._(r'videoConversion');
|
||||
static const faceDetection = QueueName._(r'faceDetection');
|
||||
static const facialRecognition = QueueName._(r'facialRecognition');
|
||||
static const smartSearch = QueueName._(r'smartSearch');
|
||||
static const duplicateDetection = QueueName._(r'duplicateDetection');
|
||||
static const backgroundTask = QueueName._(r'backgroundTask');
|
||||
static const storageTemplateMigration = QueueName._(r'storageTemplateMigration');
|
||||
static const migration = QueueName._(r'migration');
|
||||
static const search = QueueName._(r'search');
|
||||
static const sidecar = QueueName._(r'sidecar');
|
||||
static const library_ = QueueName._(r'library');
|
||||
static const notifications = QueueName._(r'notifications');
|
||||
static const backupDatabase = QueueName._(r'backupDatabase');
|
||||
static const ocr = QueueName._(r'ocr');
|
||||
|
||||
/// List of all possible values in this [enum][QueueName].
|
||||
static const values = <QueueName>[
|
||||
thumbnailGeneration,
|
||||
metadataExtraction,
|
||||
videoConversion,
|
||||
faceDetection,
|
||||
facialRecognition,
|
||||
smartSearch,
|
||||
duplicateDetection,
|
||||
backgroundTask,
|
||||
storageTemplateMigration,
|
||||
migration,
|
||||
search,
|
||||
sidecar,
|
||||
library_,
|
||||
notifications,
|
||||
backupDatabase,
|
||||
ocr,
|
||||
];
|
||||
|
||||
static QueueName? fromJson(dynamic value) => QueueNameTypeTransformer().decode(value);
|
||||
|
||||
static List<QueueName> listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <QueueName>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = QueueName.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
}
|
||||
|
||||
/// Transformation class that can [encode] an instance of [QueueName] to String,
|
||||
/// and [decode] dynamic data back to [QueueName].
|
||||
class QueueNameTypeTransformer {
|
||||
factory QueueNameTypeTransformer() => _instance ??= const QueueNameTypeTransformer._();
|
||||
|
||||
const QueueNameTypeTransformer._();
|
||||
|
||||
String encode(QueueName data) => data.value;
|
||||
|
||||
/// Decodes a [dynamic value][data] to a QueueName.
|
||||
///
|
||||
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
|
||||
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
|
||||
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
|
||||
///
|
||||
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
|
||||
/// and users are still using an old app with the old code.
|
||||
QueueName? decode(dynamic data, {bool allowNull = true}) {
|
||||
if (data != null) {
|
||||
switch (data) {
|
||||
case r'thumbnailGeneration': return QueueName.thumbnailGeneration;
|
||||
case r'metadataExtraction': return QueueName.metadataExtraction;
|
||||
case r'videoConversion': return QueueName.videoConversion;
|
||||
case r'faceDetection': return QueueName.faceDetection;
|
||||
case r'facialRecognition': return QueueName.facialRecognition;
|
||||
case r'smartSearch': return QueueName.smartSearch;
|
||||
case r'duplicateDetection': return QueueName.duplicateDetection;
|
||||
case r'backgroundTask': return QueueName.backgroundTask;
|
||||
case r'storageTemplateMigration': return QueueName.storageTemplateMigration;
|
||||
case r'migration': return QueueName.migration;
|
||||
case r'search': return QueueName.search;
|
||||
case r'sidecar': return QueueName.sidecar;
|
||||
case r'library': return QueueName.library_;
|
||||
case r'notifications': return QueueName.notifications;
|
||||
case r'backupDatabase': return QueueName.backupDatabase;
|
||||
case r'ocr': return QueueName.ocr;
|
||||
default:
|
||||
if (!allowNull) {
|
||||
throw ArgumentError('Unknown enum value to decode: $data');
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Singleton [QueueNameTypeTransformer] instance.
|
||||
static QueueNameTypeTransformer? _instance;
|
||||
}
|
||||
|
||||
@ -0,0 +1,94 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { QueueCommand, QueueName } from 'src/enum';
|
||||
import { ValidateBoolean, ValidateEnum } from 'src/validation';
|
||||
|
||||
export class QueueNameParamDto {
|
||||
@ValidateEnum({ enum: QueueName, name: 'QueueName' })
|
||||
name!: QueueName;
|
||||
}
|
||||
|
||||
export class QueueCommandDto {
|
||||
@ValidateEnum({ enum: QueueCommand, name: 'QueueCommand' })
|
||||
command!: QueueCommand;
|
||||
|
||||
@ValidateBoolean({ optional: true })
|
||||
force?: boolean; // TODO: this uses undefined as a third state, which should be refactored to be more explicit
|
||||
}
|
||||
|
||||
export class QueueStatisticsDto {
|
||||
@ApiProperty({ type: 'integer' })
|
||||
active!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
completed!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
failed!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
delayed!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
waiting!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
paused!: number;
|
||||
}
|
||||
|
||||
export class QueueStatusDto {
|
||||
isActive!: boolean;
|
||||
isPaused!: boolean;
|
||||
}
|
||||
|
||||
export class QueueResponseDto {
|
||||
@ApiProperty({ type: QueueStatisticsDto })
|
||||
jobCounts!: QueueStatisticsDto;
|
||||
|
||||
@ApiProperty({ type: QueueStatusDto })
|
||||
queueStatus!: QueueStatusDto;
|
||||
}
|
||||
|
||||
export class QueuesResponseDto implements Record<QueueName, QueueResponseDto> {
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.ThumbnailGeneration]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.MetadataExtraction]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.VideoConversion]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.SmartSearch]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.StorageTemplateMigration]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.Migration]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.BackgroundTask]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.Search]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.DuplicateDetection]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.FaceDetection]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.FacialRecognition]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.Sidecar]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.Library]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.Notification]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.BackupDatabase]!: QueueResponseDto;
|
||||
|
||||
@ApiProperty({ type: QueueResponseDto })
|
||||
[QueueName.Ocr]!: QueueResponseDto;
|
||||
}
|
||||
@ -0,0 +1,223 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { defaults, SystemConfig } from 'src/config';
|
||||
import { ImmichWorker, JobName, QueueCommand, QueueName } from 'src/enum';
|
||||
import { QueueService } from 'src/services/queue.service';
|
||||
import { newTestService, ServiceMocks } from 'test/utils';
|
||||
|
||||
describe(QueueService.name, () => {
|
||||
let sut: QueueService;
|
||||
let mocks: ServiceMocks;
|
||||
|
||||
beforeEach(() => {
|
||||
({ sut, mocks } = newTestService(QueueService));
|
||||
|
||||
mocks.config.getWorker.mockReturnValue(ImmichWorker.Microservices);
|
||||
});
|
||||
|
||||
it('should work', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('onConfigUpdate', () => {
|
||||
it('should update concurrency', () => {
|
||||
sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig });
|
||||
|
||||
expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(16);
|
||||
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(5, QueueName.FacialRecognition, 1);
|
||||
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(7, QueueName.DuplicateDetection, 1);
|
||||
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(8, QueueName.BackgroundTask, 5);
|
||||
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(9, QueueName.StorageTemplateMigration, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleNightlyJobs', () => {
|
||||
it('should run the scheduled jobs', async () => {
|
||||
await sut.handleNightlyJobs();
|
||||
|
||||
expect(mocks.job.queueAll).toHaveBeenCalledWith([
|
||||
{ name: JobName.AssetDeleteCheck },
|
||||
{ name: JobName.UserDeleteCheck },
|
||||
{ name: JobName.PersonCleanup },
|
||||
{ name: JobName.MemoryCleanup },
|
||||
{ name: JobName.SessionCleanup },
|
||||
{ name: JobName.AuditTableCleanup },
|
||||
{ name: JobName.AuditLogCleanup },
|
||||
{ name: JobName.MemoryGenerate },
|
||||
{ name: JobName.UserSyncUsage },
|
||||
{ name: JobName.AssetGenerateThumbnailsQueueAll, data: { force: false } },
|
||||
{ name: JobName.FacialRecognitionQueueAll, data: { force: false, nightly: true } },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getAllJobStatus', () => {
|
||||
it('should get all job statuses', async () => {
|
||||
mocks.job.getJobCounts.mockResolvedValue({
|
||||
active: 1,
|
||||
completed: 1,
|
||||
failed: 1,
|
||||
delayed: 1,
|
||||
waiting: 1,
|
||||
paused: 1,
|
||||
});
|
||||
mocks.job.getQueueStatus.mockResolvedValue({
|
||||
isActive: true,
|
||||
isPaused: true,
|
||||
});
|
||||
|
||||
const expectedJobStatus = {
|
||||
jobCounts: {
|
||||
active: 1,
|
||||
completed: 1,
|
||||
delayed: 1,
|
||||
failed: 1,
|
||||
waiting: 1,
|
||||
paused: 1,
|
||||
},
|
||||
queueStatus: {
|
||||
isActive: true,
|
||||
isPaused: true,
|
||||
},
|
||||
};
|
||||
|
||||
await expect(sut.getAll()).resolves.toEqual({
|
||||
[QueueName.BackgroundTask]: expectedJobStatus,
|
||||
[QueueName.DuplicateDetection]: expectedJobStatus,
|
||||
[QueueName.SmartSearch]: expectedJobStatus,
|
||||
[QueueName.MetadataExtraction]: expectedJobStatus,
|
||||
[QueueName.Search]: expectedJobStatus,
|
||||
[QueueName.StorageTemplateMigration]: expectedJobStatus,
|
||||
[QueueName.Migration]: expectedJobStatus,
|
||||
[QueueName.ThumbnailGeneration]: expectedJobStatus,
|
||||
[QueueName.VideoConversion]: expectedJobStatus,
|
||||
[QueueName.FaceDetection]: expectedJobStatus,
|
||||
[QueueName.FacialRecognition]: expectedJobStatus,
|
||||
[QueueName.Sidecar]: expectedJobStatus,
|
||||
[QueueName.Library]: expectedJobStatus,
|
||||
[QueueName.Notification]: expectedJobStatus,
|
||||
[QueueName.BackupDatabase]: expectedJobStatus,
|
||||
[QueueName.Ocr]: expectedJobStatus,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleCommand', () => {
|
||||
it('should handle a pause command', async () => {
|
||||
await sut.runCommand(QueueName.MetadataExtraction, { command: QueueCommand.Pause, force: false });
|
||||
|
||||
expect(mocks.job.pause).toHaveBeenCalledWith(QueueName.MetadataExtraction);
|
||||
});
|
||||
|
||||
it('should handle a resume command', async () => {
|
||||
await sut.runCommand(QueueName.MetadataExtraction, { command: QueueCommand.Resume, force: false });
|
||||
|
||||
expect(mocks.job.resume).toHaveBeenCalledWith(QueueName.MetadataExtraction);
|
||||
});
|
||||
|
||||
it('should handle an empty command', async () => {
|
||||
await sut.runCommand(QueueName.MetadataExtraction, { command: QueueCommand.Empty, force: false });
|
||||
|
||||
expect(mocks.job.empty).toHaveBeenCalledWith(QueueName.MetadataExtraction);
|
||||
});
|
||||
|
||||
it('should not start a job that is already running', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: true, isPaused: false });
|
||||
|
||||
await expect(
|
||||
sut.runCommand(QueueName.VideoConversion, { command: QueueCommand.Start, force: false }),
|
||||
).rejects.toBeInstanceOf(BadRequestException);
|
||||
|
||||
expect(mocks.job.queue).not.toHaveBeenCalled();
|
||||
expect(mocks.job.queueAll).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle a start video conversion command', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await sut.runCommand(QueueName.VideoConversion, { command: QueueCommand.Start, force: false });
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.AssetEncodeVideoQueueAll, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should handle a start storage template migration command', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await sut.runCommand(QueueName.StorageTemplateMigration, { command: QueueCommand.Start, force: false });
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.StorageTemplateMigration });
|
||||
});
|
||||
|
||||
it('should handle a start smart search command', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await sut.runCommand(QueueName.SmartSearch, { command: QueueCommand.Start, force: false });
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.SmartSearchQueueAll, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should handle a start metadata extraction command', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await sut.runCommand(QueueName.MetadataExtraction, { command: QueueCommand.Start, force: false });
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.AssetExtractMetadataQueueAll,
|
||||
data: { force: false },
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle a start sidecar command', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await sut.runCommand(QueueName.Sidecar, { command: QueueCommand.Start, force: false });
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.SidecarQueueAll, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should handle a start thumbnail generation command', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await sut.runCommand(QueueName.ThumbnailGeneration, { command: QueueCommand.Start, force: false });
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({
|
||||
name: JobName.AssetGenerateThumbnailsQueueAll,
|
||||
data: { force: false },
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle a start face detection command', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await sut.runCommand(QueueName.FaceDetection, { command: QueueCommand.Start, force: false });
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.AssetDetectFacesQueueAll, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should handle a start facial recognition command', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await sut.runCommand(QueueName.FacialRecognition, { command: QueueCommand.Start, force: false });
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.FacialRecognitionQueueAll, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should handle a start backup database command', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await sut.runCommand(QueueName.BackupDatabase, { command: QueueCommand.Start, force: false });
|
||||
|
||||
expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.DatabaseBackup, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should throw a bad request when an invalid queue is used', async () => {
|
||||
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
|
||||
|
||||
await expect(
|
||||
sut.runCommand(QueueName.BackgroundTask, { command: QueueCommand.Start, force: false }),
|
||||
).rejects.toBeInstanceOf(BadRequestException);
|
||||
|
||||
expect(mocks.job.queue).not.toHaveBeenCalled();
|
||||
expect(mocks.job.queueAll).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,250 @@
|
||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
||||
import { ClassConstructor } from 'class-transformer';
|
||||
import { SystemConfig } from 'src/config';
|
||||
import { OnEvent } from 'src/decorators';
|
||||
import { QueueCommandDto, QueueResponseDto, QueuesResponseDto } from 'src/dtos/queue.dto';
|
||||
import {
|
||||
BootstrapEventPriority,
|
||||
CronJob,
|
||||
DatabaseLock,
|
||||
ImmichWorker,
|
||||
JobName,
|
||||
QueueCleanType,
|
||||
QueueCommand,
|
||||
QueueName,
|
||||
} from 'src/enum';
|
||||
import { ArgOf } from 'src/repositories/event.repository';
|
||||
import { BaseService } from 'src/services/base.service';
|
||||
import { ConcurrentQueueName, JobItem } from 'src/types';
|
||||
import { handlePromiseError } from 'src/utils/misc';
|
||||
|
||||
const asNightlyTasksCron = (config: SystemConfig) => {
|
||||
const [hours, minutes] = config.nightlyTasks.startTime.split(':').map(Number);
|
||||
return `${minutes} ${hours} * * *`;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class QueueService extends BaseService {
|
||||
private services: ClassConstructor<unknown>[] = [];
|
||||
private nightlyJobsLock = false;
|
||||
|
||||
@OnEvent({ name: 'ConfigInit' })
|
||||
async onConfigInit({ newConfig: config }: ArgOf<'ConfigInit'>) {
|
||||
if (this.worker === ImmichWorker.Microservices) {
|
||||
this.updateConcurrency(config);
|
||||
return;
|
||||
}
|
||||
|
||||
this.nightlyJobsLock = await this.databaseRepository.tryLock(DatabaseLock.NightlyJobs);
|
||||
if (this.nightlyJobsLock) {
|
||||
const cronExpression = asNightlyTasksCron(config);
|
||||
this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
|
||||
this.cronRepository.create({
|
||||
name: CronJob.NightlyJobs,
|
||||
expression: cronExpression,
|
||||
start: true,
|
||||
onTick: () => handlePromiseError(this.handleNightlyJobs(), this.logger),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@OnEvent({ name: 'ConfigUpdate', server: true })
|
||||
onConfigUpdate({ newConfig: config }: ArgOf<'ConfigUpdate'>) {
|
||||
if (this.worker === ImmichWorker.Microservices) {
|
||||
this.updateConcurrency(config);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.nightlyJobsLock) {
|
||||
const cronExpression = asNightlyTasksCron(config);
|
||||
this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
|
||||
this.cronRepository.update({ name: CronJob.NightlyJobs, expression: cronExpression, start: true });
|
||||
}
|
||||
}
|
||||
|
||||
@OnEvent({ name: 'AppBootstrap', priority: BootstrapEventPriority.JobService })
|
||||
onBootstrap() {
|
||||
this.jobRepository.setup(this.services);
|
||||
if (this.worker === ImmichWorker.Microservices) {
|
||||
this.jobRepository.startWorkers();
|
||||
}
|
||||
}
|
||||
|
||||
private updateConcurrency(config: SystemConfig) {
|
||||
this.logger.debug(`Updating queue concurrency settings`);
|
||||
for (const queueName of Object.values(QueueName)) {
|
||||
let concurrency = 1;
|
||||
if (this.isConcurrentQueue(queueName)) {
|
||||
concurrency = config.job[queueName].concurrency;
|
||||
}
|
||||
this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
|
||||
this.jobRepository.setConcurrency(queueName, concurrency);
|
||||
}
|
||||
}
|
||||
|
||||
setServices(services: ClassConstructor<unknown>[]) {
|
||||
this.services = services;
|
||||
}
|
||||
|
||||
async runCommand(name: QueueName, dto: QueueCommandDto): Promise<QueueResponseDto> {
|
||||
this.logger.debug(`Handling command: queue=${name},command=${dto.command},force=${dto.force}`);
|
||||
|
||||
switch (dto.command) {
|
||||
case QueueCommand.Start: {
|
||||
await this.start(name, dto);
|
||||
break;
|
||||
}
|
||||
|
||||
case QueueCommand.Pause: {
|
||||
await this.jobRepository.pause(name);
|
||||
break;
|
||||
}
|
||||
|
||||
case QueueCommand.Resume: {
|
||||
await this.jobRepository.resume(name);
|
||||
break;
|
||||
}
|
||||
|
||||
case QueueCommand.Empty: {
|
||||
await this.jobRepository.empty(name);
|
||||
break;
|
||||
}
|
||||
|
||||
case QueueCommand.ClearFailed: {
|
||||
const failedJobs = await this.jobRepository.clear(name, QueueCleanType.Failed);
|
||||
this.logger.debug(`Cleared failed jobs: ${failedJobs}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return this.getByName(name);
|
||||
}
|
||||
|
||||
async getAll(): Promise<QueuesResponseDto> {
|
||||
const response = new QueuesResponseDto();
|
||||
for (const name of Object.values(QueueName)) {
|
||||
response[name] = await this.getByName(name);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async getByName(name: QueueName): Promise<QueueResponseDto> {
|
||||
const [jobCounts, queueStatus] = await Promise.all([
|
||||
this.jobRepository.getJobCounts(name),
|
||||
this.jobRepository.getQueueStatus(name),
|
||||
]);
|
||||
|
||||
return { jobCounts, queueStatus };
|
||||
}
|
||||
|
||||
private async start(name: QueueName, { force }: QueueCommandDto): Promise<void> {
|
||||
const { isActive } = await this.jobRepository.getQueueStatus(name);
|
||||
if (isActive) {
|
||||
throw new BadRequestException(`Job is already running`);
|
||||
}
|
||||
|
||||
await this.eventRepository.emit('QueueStart', { name });
|
||||
|
||||
switch (name) {
|
||||
case QueueName.VideoConversion: {
|
||||
return this.jobRepository.queue({ name: JobName.AssetEncodeVideoQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.StorageTemplateMigration: {
|
||||
return this.jobRepository.queue({ name: JobName.StorageTemplateMigration });
|
||||
}
|
||||
|
||||
case QueueName.Migration: {
|
||||
return this.jobRepository.queue({ name: JobName.FileMigrationQueueAll });
|
||||
}
|
||||
|
||||
case QueueName.SmartSearch: {
|
||||
return this.jobRepository.queue({ name: JobName.SmartSearchQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.DuplicateDetection: {
|
||||
return this.jobRepository.queue({ name: JobName.AssetDetectDuplicatesQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.MetadataExtraction: {
|
||||
return this.jobRepository.queue({ name: JobName.AssetExtractMetadataQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.Sidecar: {
|
||||
return this.jobRepository.queue({ name: JobName.SidecarQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.ThumbnailGeneration: {
|
||||
return this.jobRepository.queue({ name: JobName.AssetGenerateThumbnailsQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.FaceDetection: {
|
||||
return this.jobRepository.queue({ name: JobName.AssetDetectFacesQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.FacialRecognition: {
|
||||
return this.jobRepository.queue({ name: JobName.FacialRecognitionQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.Library: {
|
||||
return this.jobRepository.queue({ name: JobName.LibraryScanQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.BackupDatabase: {
|
||||
return this.jobRepository.queue({ name: JobName.DatabaseBackup, data: { force } });
|
||||
}
|
||||
|
||||
case QueueName.Ocr: {
|
||||
return this.jobRepository.queue({ name: JobName.OcrQueueAll, data: { force } });
|
||||
}
|
||||
|
||||
default: {
|
||||
throw new BadRequestException(`Invalid job name: ${name}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private isConcurrentQueue(name: QueueName): name is ConcurrentQueueName {
|
||||
return ![
|
||||
QueueName.FacialRecognition,
|
||||
QueueName.StorageTemplateMigration,
|
||||
QueueName.DuplicateDetection,
|
||||
QueueName.BackupDatabase,
|
||||
].includes(name);
|
||||
}
|
||||
|
||||
async handleNightlyJobs() {
|
||||
const config = await this.getConfig({ withCache: false });
|
||||
const jobs: JobItem[] = [];
|
||||
|
||||
if (config.nightlyTasks.databaseCleanup) {
|
||||
jobs.push(
|
||||
{ name: JobName.AssetDeleteCheck },
|
||||
{ name: JobName.UserDeleteCheck },
|
||||
{ name: JobName.PersonCleanup },
|
||||
{ name: JobName.MemoryCleanup },
|
||||
{ name: JobName.SessionCleanup },
|
||||
{ name: JobName.AuditTableCleanup },
|
||||
{ name: JobName.AuditLogCleanup },
|
||||
);
|
||||
}
|
||||
|
||||
if (config.nightlyTasks.generateMemories) {
|
||||
jobs.push({ name: JobName.MemoryGenerate });
|
||||
}
|
||||
|
||||
if (config.nightlyTasks.syncQuotaUsage) {
|
||||
jobs.push({ name: JobName.UserSyncUsage });
|
||||
}
|
||||
|
||||
if (config.nightlyTasks.missingThumbnails) {
|
||||
jobs.push({ name: JobName.AssetGenerateThumbnailsQueueAll, data: { force: false } });
|
||||
}
|
||||
|
||||
if (config.nightlyTasks.clusterNewFaces) {
|
||||
jobs.push({ name: JobName.FacialRecognitionQueueAll, data: { force: false, nightly: true } });
|
||||
}
|
||||
|
||||
await this.jobRepository.queueAll(jobs);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue