mirror of https://github.com/immich-app/immich.git
refactor(server): jobs (#2023)
* refactor: job to domain * chore: regenerate open api * chore: tests * fix: missing breaks * fix: get asset with missing exif data --------- Co-authored-by: Alex Tran <alex.tran1502@gmail.com>pull/2024/head
Jason Rasmussen
GitHub
parent
db6b14361d
commit
386eef046d
@ -1,4 +1,4 @@
|
||||
# openapi.model.JobCounts
|
||||
# openapi.model.JobCountsDto
|
||||
|
||||
## Load the model package
|
||||
```dart
|
||||
@ -1,4 +1,4 @@
|
||||
# openapi.model.JobId
|
||||
# openapi.model.JobName
|
||||
|
||||
## Load the model package
|
||||
```dart
|
||||
@ -1,94 +0,0 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
|
||||
class JobId {
|
||||
/// Instantiate a new enum with the provided [value].
|
||||
const JobId._(this.value);
|
||||
|
||||
/// The underlying value of this enum member.
|
||||
final String value;
|
||||
|
||||
@override
|
||||
String toString() => value;
|
||||
|
||||
String toJson() => value;
|
||||
|
||||
static const thumbnailGeneration = JobId._(r'thumbnail-generation');
|
||||
static const metadataExtraction = JobId._(r'metadata-extraction');
|
||||
static const videoConversion = JobId._(r'video-conversion');
|
||||
static const machineLearning = JobId._(r'machine-learning');
|
||||
static const storageTemplateMigration = JobId._(r'storage-template-migration');
|
||||
|
||||
/// List of all possible values in this [enum][JobId].
|
||||
static const values = <JobId>[
|
||||
thumbnailGeneration,
|
||||
metadataExtraction,
|
||||
videoConversion,
|
||||
machineLearning,
|
||||
storageTemplateMigration,
|
||||
];
|
||||
|
||||
static JobId? fromJson(dynamic value) => JobIdTypeTransformer().decode(value);
|
||||
|
||||
static List<JobId>? listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <JobId>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = JobId.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
}
|
||||
|
||||
/// Transformation class that can [encode] an instance of [JobId] to String,
|
||||
/// and [decode] dynamic data back to [JobId].
|
||||
class JobIdTypeTransformer {
|
||||
factory JobIdTypeTransformer() => _instance ??= const JobIdTypeTransformer._();
|
||||
|
||||
const JobIdTypeTransformer._();
|
||||
|
||||
String encode(JobId data) => data.value;
|
||||
|
||||
/// Decodes a [dynamic value][data] to a JobId.
|
||||
///
|
||||
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
|
||||
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
|
||||
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
|
||||
///
|
||||
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
|
||||
/// and users are still using an old app with the old code.
|
||||
JobId? decode(dynamic data, {bool allowNull = true}) {
|
||||
if (data != null) {
|
||||
switch (data.toString()) {
|
||||
case r'thumbnail-generation': return JobId.thumbnailGeneration;
|
||||
case r'metadata-extraction': return JobId.metadataExtraction;
|
||||
case r'video-conversion': return JobId.videoConversion;
|
||||
case r'machine-learning': return JobId.machineLearning;
|
||||
case r'storage-template-migration': return JobId.storageTemplateMigration;
|
||||
default:
|
||||
if (!allowNull) {
|
||||
throw ArgumentError('Unknown enum value to decode: $data');
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Singleton [JobIdTypeTransformer] instance.
|
||||
static JobIdTypeTransformer? _instance;
|
||||
}
|
||||
|
||||
@ -0,0 +1,103 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
|
||||
class JobName {
|
||||
/// Instantiate a new enum with the provided [value].
|
||||
const JobName._(this.value);
|
||||
|
||||
/// The underlying value of this enum member.
|
||||
final String value;
|
||||
|
||||
@override
|
||||
String toString() => value;
|
||||
|
||||
String toJson() => value;
|
||||
|
||||
static const thumbnailGenerationQueue = JobName._(r'thumbnail-generation-queue');
|
||||
static const metadataExtractionQueue = JobName._(r'metadata-extraction-queue');
|
||||
static const videoConversionQueue = JobName._(r'video-conversion-queue');
|
||||
static const objectTaggingQueue = JobName._(r'object-tagging-queue');
|
||||
static const clipEncodingQueue = JobName._(r'clip-encoding-queue');
|
||||
static const backgroundTaskQueue = JobName._(r'background-task-queue');
|
||||
static const storageTemplateMigrationQueue = JobName._(r'storage-template-migration-queue');
|
||||
static const searchQueue = JobName._(r'search-queue');
|
||||
|
||||
/// List of all possible values in this [enum][JobName].
|
||||
static const values = <JobName>[
|
||||
thumbnailGenerationQueue,
|
||||
metadataExtractionQueue,
|
||||
videoConversionQueue,
|
||||
objectTaggingQueue,
|
||||
clipEncodingQueue,
|
||||
backgroundTaskQueue,
|
||||
storageTemplateMigrationQueue,
|
||||
searchQueue,
|
||||
];
|
||||
|
||||
static JobName? fromJson(dynamic value) => JobNameTypeTransformer().decode(value);
|
||||
|
||||
static List<JobName>? listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <JobName>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = JobName.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
}
|
||||
|
||||
/// Transformation class that can [encode] an instance of [JobName] to String,
|
||||
/// and [decode] dynamic data back to [JobName].
|
||||
class JobNameTypeTransformer {
|
||||
factory JobNameTypeTransformer() => _instance ??= const JobNameTypeTransformer._();
|
||||
|
||||
const JobNameTypeTransformer._();
|
||||
|
||||
String encode(JobName data) => data.value;
|
||||
|
||||
/// Decodes a [dynamic value][data] to a JobName.
|
||||
///
|
||||
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
|
||||
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
|
||||
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
|
||||
///
|
||||
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
|
||||
/// and users are still using an old app with the old code.
|
||||
JobName? decode(dynamic data, {bool allowNull = true}) {
|
||||
if (data != null) {
|
||||
switch (data.toString()) {
|
||||
case r'thumbnail-generation-queue': return JobName.thumbnailGenerationQueue;
|
||||
case r'metadata-extraction-queue': return JobName.metadataExtractionQueue;
|
||||
case r'video-conversion-queue': return JobName.videoConversionQueue;
|
||||
case r'object-tagging-queue': return JobName.objectTaggingQueue;
|
||||
case r'clip-encoding-queue': return JobName.clipEncodingQueue;
|
||||
case r'background-task-queue': return JobName.backgroundTaskQueue;
|
||||
case r'storage-template-migration-queue': return JobName.storageTemplateMigrationQueue;
|
||||
case r'search-queue': return JobName.searchQueue;
|
||||
default:
|
||||
if (!allowNull) {
|
||||
throw ArgumentError('Unknown enum value to decode: $data');
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Singleton [JobNameTypeTransformer] instance.
|
||||
static JobNameTypeTransformer? _instance;
|
||||
}
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { IsEnum, IsNotEmpty } from 'class-validator';
|
||||
|
||||
export enum JobId {
|
||||
THUMBNAIL_GENERATION = 'thumbnail-generation',
|
||||
METADATA_EXTRACTION = 'metadata-extraction',
|
||||
VIDEO_CONVERSION = 'video-conversion',
|
||||
MACHINE_LEARNING = 'machine-learning',
|
||||
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration',
|
||||
}
|
||||
|
||||
export class GetJobDto {
|
||||
@IsNotEmpty()
|
||||
@IsEnum(JobId, {
|
||||
message: `params must be one of ${Object.values(JobId).join()}`,
|
||||
})
|
||||
@ApiProperty({
|
||||
type: String,
|
||||
enum: JobId,
|
||||
enumName: 'JobId',
|
||||
})
|
||||
jobId!: JobId;
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { IsBoolean, IsIn, IsNotEmpty, IsOptional } from 'class-validator';
|
||||
|
||||
export class JobCommandDto {
|
||||
@IsNotEmpty()
|
||||
@IsIn(['start', 'stop'])
|
||||
@ApiProperty({
|
||||
enum: ['start', 'stop'],
|
||||
enumName: 'JobCommand',
|
||||
})
|
||||
command!: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
includeAllAssets!: boolean;
|
||||
}
|
||||
@ -1,33 +0,0 @@
|
||||
import { Body, Controller, Get, Param, Put, ValidationPipe } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import { Authenticated } from '../../decorators/authenticated.decorator';
|
||||
import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto';
|
||||
import { GetJobDto } from './dto/get-job.dto';
|
||||
import { JobService } from './job.service';
|
||||
import { JobCommandDto } from './dto/job-command.dto';
|
||||
|
||||
@Authenticated({ admin: true })
|
||||
@ApiTags('Job')
|
||||
@Controller('jobs')
|
||||
export class JobController {
|
||||
constructor(private readonly jobService: JobService) {}
|
||||
|
||||
@Get()
|
||||
getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
|
||||
return this.jobService.getAllJobsStatus();
|
||||
}
|
||||
|
||||
@Put('/:jobId')
|
||||
async sendJobCommand(
|
||||
@Param(ValidationPipe) params: GetJobDto,
|
||||
@Body(ValidationPipe) dto: JobCommandDto,
|
||||
): Promise<number> {
|
||||
if (dto.command === 'start') {
|
||||
return await this.jobService.start(params.jobId, dto.includeAllAssets);
|
||||
}
|
||||
if (dto.command === 'stop') {
|
||||
return await this.jobService.stop(params.jobId);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { JobService } from './job.service';
|
||||
import { JobController } from './job.controller';
|
||||
import { AssetModule } from '../asset/asset.module';
|
||||
|
||||
@Module({
|
||||
imports: [AssetModule],
|
||||
controllers: [JobController],
|
||||
providers: [JobService],
|
||||
})
|
||||
export class JobModule {}
|
||||
@ -1,142 +0,0 @@
|
||||
import { JobName, IJobRepository, QueueName } from '@app/domain';
|
||||
import { BadRequestException, Inject, Injectable } from '@nestjs/common';
|
||||
import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto';
|
||||
import { IAssetRepository } from '../asset/asset-repository';
|
||||
import { AssetType } from '@app/infra';
|
||||
import { JobId } from './dto/get-job.dto';
|
||||
import { MACHINE_LEARNING_ENABLED } from '@app/common';
|
||||
import { getFileNameWithoutExtension } from '../../utils/file-name.util';
|
||||
const jobIds = Object.values(JobId) as JobId[];
|
||||
|
||||
@Injectable()
|
||||
export class JobService {
|
||||
constructor(
|
||||
@Inject(IAssetRepository) private _assetRepository: IAssetRepository,
|
||||
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
||||
) {
|
||||
for (const jobId of jobIds) {
|
||||
this.jobRepository.empty(this.asQueueName(jobId));
|
||||
}
|
||||
}
|
||||
|
||||
start(jobId: JobId, includeAllAssets: boolean): Promise<number> {
|
||||
return this.run(this.asQueueName(jobId), includeAllAssets);
|
||||
}
|
||||
|
||||
async stop(jobId: JobId): Promise<number> {
|
||||
await this.jobRepository.empty(this.asQueueName(jobId));
|
||||
return 0;
|
||||
}
|
||||
|
||||
async getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
|
||||
const response = new AllJobStatusResponseDto();
|
||||
for (const jobId of jobIds) {
|
||||
response[jobId] = await this.jobRepository.getJobCounts(this.asQueueName(jobId));
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
private async run(name: QueueName, includeAllAssets: boolean): Promise<number> {
|
||||
const isActive = await this.jobRepository.isActive(name);
|
||||
if (isActive) {
|
||||
throw new BadRequestException(`Job is already running`);
|
||||
}
|
||||
|
||||
switch (name) {
|
||||
case QueueName.VIDEO_CONVERSION: {
|
||||
const assets = includeAllAssets
|
||||
? await this._assetRepository.getAllVideos()
|
||||
: await this._assetRepository.getAssetWithNoEncodedVideo();
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.queue({ name: JobName.VIDEO_CONVERSION, data: { asset } });
|
||||
}
|
||||
|
||||
return assets.length;
|
||||
}
|
||||
|
||||
case QueueName.STORAGE_TEMPLATE_MIGRATION:
|
||||
await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
|
||||
return 1;
|
||||
|
||||
case QueueName.MACHINE_LEARNING: {
|
||||
if (!MACHINE_LEARNING_ENABLED) {
|
||||
throw new BadRequestException('Machine learning is not enabled.');
|
||||
}
|
||||
|
||||
const assets = includeAllAssets
|
||||
? await this._assetRepository.getAll()
|
||||
: await this._assetRepository.getAssetWithNoSmartInfo();
|
||||
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.queue({ name: JobName.IMAGE_TAGGING, data: { asset } });
|
||||
await this.jobRepository.queue({ name: JobName.OBJECT_DETECTION, data: { asset } });
|
||||
await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: { asset } });
|
||||
}
|
||||
return assets.length;
|
||||
}
|
||||
|
||||
case QueueName.METADATA_EXTRACTION: {
|
||||
const assets = includeAllAssets
|
||||
? await this._assetRepository.getAll()
|
||||
: await this._assetRepository.getAssetWithNoEXIF();
|
||||
|
||||
for (const asset of assets) {
|
||||
if (asset.type === AssetType.VIDEO) {
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.EXTRACT_VIDEO_METADATA,
|
||||
data: {
|
||||
asset,
|
||||
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
|
||||
},
|
||||
});
|
||||
} else {
|
||||
await this.jobRepository.queue({
|
||||
name: JobName.EXIF_EXTRACTION,
|
||||
data: {
|
||||
asset,
|
||||
fileName: asset.exifInfo?.imageName ?? getFileNameWithoutExtension(asset.originalPath),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
return assets.length;
|
||||
}
|
||||
|
||||
case QueueName.THUMBNAIL_GENERATION: {
|
||||
const assets = includeAllAssets
|
||||
? await this._assetRepository.getAll()
|
||||
: await this._assetRepository.getAssetWithNoThumbnail();
|
||||
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
|
||||
}
|
||||
return assets.length;
|
||||
}
|
||||
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private asQueueName(jobId: JobId) {
|
||||
switch (jobId) {
|
||||
case JobId.THUMBNAIL_GENERATION:
|
||||
return QueueName.THUMBNAIL_GENERATION;
|
||||
|
||||
case JobId.METADATA_EXTRACTION:
|
||||
return QueueName.METADATA_EXTRACTION;
|
||||
|
||||
case JobId.VIDEO_CONVERSION:
|
||||
return QueueName.VIDEO_CONVERSION;
|
||||
|
||||
case JobId.STORAGE_TEMPLATE_MIGRATION:
|
||||
return QueueName.STORAGE_TEMPLATE_MIGRATION;
|
||||
|
||||
case JobId.MACHINE_LEARNING:
|
||||
return QueueName.MACHINE_LEARNING;
|
||||
|
||||
default:
|
||||
throw new BadRequestException(`Invalid job id: ${jobId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { JobId } from '../dto/get-job.dto';
|
||||
|
||||
export class JobCounts {
|
||||
@ApiProperty({ type: 'integer' })
|
||||
active!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
completed!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
failed!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
delayed!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
waiting!: number;
|
||||
}
|
||||
|
||||
export class AllJobStatusResponseDto {
|
||||
@ApiProperty({ type: JobCounts })
|
||||
[JobId.THUMBNAIL_GENERATION]!: JobCounts;
|
||||
|
||||
@ApiProperty({ type: JobCounts })
|
||||
[JobId.METADATA_EXTRACTION]!: JobCounts;
|
||||
|
||||
@ApiProperty({ type: JobCounts })
|
||||
[JobId.VIDEO_CONVERSION]!: JobCounts;
|
||||
|
||||
@ApiProperty({ type: JobCounts })
|
||||
[JobId.MACHINE_LEARNING]!: JobCounts;
|
||||
|
||||
@ApiProperty({ type: JobCounts })
|
||||
[JobId.STORAGE_TEMPLATE_MIGRATION]!: JobCounts;
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
import { AllJobStatusResponseDto, JobCommandDto, JobIdDto, JobService } from '@app/domain';
|
||||
import { Body, Controller, Get, Param, Put, ValidationPipe } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import { Authenticated } from '../decorators/authenticated.decorator';
|
||||
|
||||
@Authenticated({ admin: true })
|
||||
@ApiTags('Job')
|
||||
@Controller('jobs')
|
||||
export class JobController {
|
||||
constructor(private readonly jobService: JobService) {}
|
||||
|
||||
@Get()
|
||||
getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
|
||||
return this.jobService.getAllJobsStatus();
|
||||
}
|
||||
|
||||
@Put('/:jobId')
|
||||
sendJobCommand(@Param(ValidationPipe) { jobId }: JobIdDto, @Body(ValidationPipe) dto: JobCommandDto): Promise<void> {
|
||||
return this.jobService.handleCommand(jobId, dto);
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,12 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
|
||||
export * from './upload_location.constant';
|
||||
|
||||
export const MACHINE_LEARNING_URL = process.env.IMMICH_MACHINE_LEARNING_URL || 'http://immich-machine-learning:3003';
|
||||
export const MACHINE_LEARNING_ENABLED = MACHINE_LEARNING_URL !== 'false';
|
||||
|
||||
export function assertMachineLearningEnabled() {
|
||||
if (!MACHINE_LEARNING_ENABLED) {
|
||||
throw new BadRequestException('Machine learning is not enabled.');
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
export * from './job-command.dto';
|
||||
export * from './job-id.dto';
|
||||
@ -0,0 +1,14 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { IsBoolean, IsEnum, IsNotEmpty, IsOptional } from 'class-validator';
|
||||
import { JobCommand } from '../job.constants';
|
||||
|
||||
export class JobCommandDto {
|
||||
@IsNotEmpty()
|
||||
@IsEnum(JobCommand)
|
||||
@ApiProperty({ type: 'string', enum: JobCommand, enumName: 'JobCommand' })
|
||||
command!: JobCommand;
|
||||
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
force!: boolean;
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { IsEnum, IsNotEmpty } from 'class-validator';
|
||||
import { QueueName } from '../job.constants';
|
||||
|
||||
export class JobIdDto {
|
||||
@IsNotEmpty()
|
||||
@IsEnum(QueueName)
|
||||
@ApiProperty({ type: String, enum: QueueName, enumName: 'JobName' })
|
||||
jobId!: QueueName;
|
||||
}
|
||||
@ -1,3 +1,6 @@
|
||||
export * from './dto';
|
||||
export * from './job.constants';
|
||||
export * from './job.interface';
|
||||
export * from './job.repository';
|
||||
export * from './job.service';
|
||||
export * from './response-dto';
|
||||
|
||||
@ -0,0 +1,170 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { newJobRepositoryMock } from '../../test';
|
||||
import { IJobRepository, JobCommand, JobName, JobService, QueueName } from '../job';
|
||||
|
||||
describe(JobService.name, () => {
|
||||
let sut: JobService;
|
||||
let jobMock: jest.Mocked<IJobRepository>;
|
||||
|
||||
beforeEach(async () => {
|
||||
jobMock = newJobRepositoryMock();
|
||||
sut = new JobService(jobMock);
|
||||
});
|
||||
|
||||
it('should work', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('getAllJobStatus', () => {
|
||||
it('should get all job statuses', async () => {
|
||||
jobMock.getJobCounts.mockResolvedValue({
|
||||
active: 1,
|
||||
completed: 1,
|
||||
failed: 1,
|
||||
delayed: 1,
|
||||
waiting: 1,
|
||||
});
|
||||
|
||||
await expect(sut.getAllJobsStatus()).resolves.toEqual({
|
||||
'background-task-queue': {
|
||||
active: 1,
|
||||
completed: 1,
|
||||
delayed: 1,
|
||||
failed: 1,
|
||||
waiting: 1,
|
||||
},
|
||||
'clip-encoding-queue': {
|
||||
active: 1,
|
||||
completed: 1,
|
||||
delayed: 1,
|
||||
failed: 1,
|
||||
waiting: 1,
|
||||
},
|
||||
'metadata-extraction-queue': {
|
||||
active: 1,
|
||||
completed: 1,
|
||||
delayed: 1,
|
||||
failed: 1,
|
||||
waiting: 1,
|
||||
},
|
||||
'object-tagging-queue': {
|
||||
active: 1,
|
||||
completed: 1,
|
||||
delayed: 1,
|
||||
failed: 1,
|
||||
waiting: 1,
|
||||
},
|
||||
'search-queue': {
|
||||
active: 1,
|
||||
completed: 1,
|
||||
delayed: 1,
|
||||
failed: 1,
|
||||
waiting: 1,
|
||||
},
|
||||
'storage-template-migration-queue': {
|
||||
active: 1,
|
||||
completed: 1,
|
||||
delayed: 1,
|
||||
failed: 1,
|
||||
waiting: 1,
|
||||
},
|
||||
'thumbnail-generation-queue': {
|
||||
active: 1,
|
||||
completed: 1,
|
||||
delayed: 1,
|
||||
failed: 1,
|
||||
waiting: 1,
|
||||
},
|
||||
'video-conversion-queue': {
|
||||
active: 1,
|
||||
completed: 1,
|
||||
delayed: 1,
|
||||
failed: 1,
|
||||
waiting: 1,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleCommand', () => {
|
||||
it('should handle a pause command', async () => {
|
||||
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.PAUSE, force: false });
|
||||
|
||||
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
|
||||
});
|
||||
|
||||
it('should handle an empty command', async () => {
|
||||
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.EMPTY, force: false });
|
||||
|
||||
expect(jobMock.empty).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
|
||||
});
|
||||
|
||||
it('should not start a job that is already running', async () => {
|
||||
jobMock.isActive.mockResolvedValue(true);
|
||||
|
||||
await expect(
|
||||
sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false }),
|
||||
).rejects.toBeInstanceOf(BadRequestException);
|
||||
|
||||
expect(jobMock.queue).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should handle a start video conversion command', async () => {
|
||||
jobMock.isActive.mockResolvedValue(false);
|
||||
|
||||
await sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false });
|
||||
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_VIDEO_CONVERSION, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should handle a start storage template migration command', async () => {
|
||||
jobMock.isActive.mockResolvedValue(false);
|
||||
|
||||
await sut.handleCommand(QueueName.STORAGE_TEMPLATE_MIGRATION, { command: JobCommand.START, force: false });
|
||||
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
|
||||
});
|
||||
|
||||
it('should handle a start object tagging command', async () => {
|
||||
jobMock.isActive.mockResolvedValue(false);
|
||||
|
||||
await sut.handleCommand(QueueName.OBJECT_TAGGING, { command: JobCommand.START, force: false });
|
||||
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_OBJECT_TAGGING, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should handle a start clip encoding command', async () => {
|
||||
jobMock.isActive.mockResolvedValue(false);
|
||||
|
||||
await sut.handleCommand(QueueName.CLIP_ENCODING, { command: JobCommand.START, force: false });
|
||||
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_ENCODE_CLIP, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should handle a start metadata extraction command', async () => {
|
||||
jobMock.isActive.mockResolvedValue(false);
|
||||
|
||||
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.START, force: false });
|
||||
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should handle a start thumbnail generation command', async () => {
|
||||
jobMock.isActive.mockResolvedValue(false);
|
||||
|
||||
await sut.handleCommand(QueueName.THUMBNAIL_GENERATION, { command: JobCommand.START, force: false });
|
||||
|
||||
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
|
||||
});
|
||||
|
||||
it('should throw a bad request when an invalid queue is used', async () => {
|
||||
jobMock.isActive.mockResolvedValue(false);
|
||||
|
||||
await expect(
|
||||
sut.handleCommand(QueueName.BACKGROUND_TASK, { command: JobCommand.START, force: false }),
|
||||
).rejects.toBeInstanceOf(BadRequestException);
|
||||
|
||||
expect(jobMock.queue).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,68 @@
|
||||
import { assertMachineLearningEnabled } from '@app/common';
|
||||
import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import { JobCommandDto } from './dto';
|
||||
import { JobCommand, JobName, QueueName } from './job.constants';
|
||||
import { IJobRepository } from './job.repository';
|
||||
import { AllJobStatusResponseDto } from './response-dto';
|
||||
|
||||
@Injectable()
|
||||
export class JobService {
|
||||
private logger = new Logger(JobService.name);
|
||||
|
||||
constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {}
|
||||
|
||||
handleCommand(queueName: QueueName, dto: JobCommandDto): Promise<void> {
|
||||
this.logger.debug(`Handling command: queue=${queueName},force=${dto.force}`);
|
||||
|
||||
switch (dto.command) {
|
||||
case JobCommand.START:
|
||||
return this.start(queueName, dto);
|
||||
|
||||
case JobCommand.PAUSE:
|
||||
return this.jobRepository.pause(queueName);
|
||||
|
||||
case JobCommand.EMPTY:
|
||||
return this.jobRepository.empty(queueName);
|
||||
}
|
||||
}
|
||||
|
||||
async getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
|
||||
const response = new AllJobStatusResponseDto();
|
||||
for (const queueName of Object.values(QueueName)) {
|
||||
response[queueName] = await this.jobRepository.getJobCounts(queueName);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
private async start(name: QueueName, { force }: JobCommandDto): Promise<void> {
|
||||
const isActive = await this.jobRepository.isActive(name);
|
||||
if (isActive) {
|
||||
throw new BadRequestException(`Job is already running`);
|
||||
}
|
||||
|
||||
switch (name) {
|
||||
case QueueName.VIDEO_CONVERSION:
|
||||
return this.jobRepository.queue({ name: JobName.QUEUE_VIDEO_CONVERSION, data: { force } });
|
||||
|
||||
case QueueName.STORAGE_TEMPLATE_MIGRATION:
|
||||
return this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
|
||||
|
||||
case QueueName.OBJECT_TAGGING:
|
||||
assertMachineLearningEnabled();
|
||||
return this.jobRepository.queue({ name: JobName.QUEUE_OBJECT_TAGGING, data: { force } });
|
||||
|
||||
case QueueName.CLIP_ENCODING:
|
||||
assertMachineLearningEnabled();
|
||||
return this.jobRepository.queue({ name: JobName.QUEUE_ENCODE_CLIP, data: { force } });
|
||||
|
||||
case QueueName.METADATA_EXTRACTION:
|
||||
return this.jobRepository.queue({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force } });
|
||||
|
||||
case QueueName.THUMBNAIL_GENERATION:
|
||||
return this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force } });
|
||||
|
||||
default:
|
||||
throw new BadRequestException(`Invalid job name: ${name}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { QueueName } from '../job.constants';
|
||||
|
||||
export class JobCountsDto {
|
||||
@ApiProperty({ type: 'integer' })
|
||||
active!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
completed!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
failed!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
delayed!: number;
|
||||
@ApiProperty({ type: 'integer' })
|
||||
waiting!: number;
|
||||
}
|
||||
|
||||
export class AllJobStatusResponseDto implements Record<QueueName, JobCountsDto> {
|
||||
@ApiProperty({ type: JobCountsDto })
|
||||
[QueueName.THUMBNAIL_GENERATION]!: JobCountsDto;
|
||||
|
||||
@ApiProperty({ type: JobCountsDto })
|
||||
[QueueName.METADATA_EXTRACTION]!: JobCountsDto;
|
||||
|
||||
@ApiProperty({ type: JobCountsDto })
|
||||
[QueueName.VIDEO_CONVERSION]!: JobCountsDto;
|
||||
|
||||
@ApiProperty({ type: JobCountsDto })
|
||||
[QueueName.OBJECT_TAGGING]!: JobCountsDto;
|
||||
|
||||
@ApiProperty({ type: JobCountsDto })
|
||||
[QueueName.CLIP_ENCODING]!: JobCountsDto;
|
||||
|
||||
@ApiProperty({ type: JobCountsDto })
|
||||
[QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobCountsDto;
|
||||
|
||||
@ApiProperty({ type: JobCountsDto })
|
||||
[QueueName.BACKGROUND_TASK]!: JobCountsDto;
|
||||
|
||||
@ApiProperty({ type: JobCountsDto })
|
||||
[QueueName.SEARCH]!: JobCountsDto;
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
export * from './all-job-status-response.dto';
|
||||
Loading…
Reference in New Issue