mirror of https://github.com/immich-app/immich.git
refactor(server): job repository (#1382)
* refactor(server): job repository * refactor: job repository * chore: generate open-api * fix: job panel * Remove incorrect subtitle Co-authored-by: Alex Tran <alex.tran1502@gmail.com>pull/1384/head
Jason Rasmussen
GitHub
parent
f4c90426a5
commit
4cfac47674
@ -1,16 +0,0 @@
|
||||
# openapi.model.JobStatusResponseDto
|
||||
|
||||
## Load the model package
|
||||
```dart
|
||||
import 'package:openapi/api.dart';
|
||||
```
|
||||
|
||||
## Properties
|
||||
Name | Type | Description | Notes
|
||||
------------ | ------------- | ------------- | -------------
|
||||
**isActive** | **bool** | |
|
||||
**queueCount** | [**Object**](.md) | |
|
||||
|
||||
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||
|
||||
|
||||
@ -1,119 +0,0 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
class JobStatusResponseDto {
|
||||
/// Returns a new [JobStatusResponseDto] instance.
|
||||
JobStatusResponseDto({
|
||||
required this.isActive,
|
||||
required this.queueCount,
|
||||
});
|
||||
|
||||
bool isActive;
|
||||
|
||||
Object queueCount;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) => identical(this, other) || other is JobStatusResponseDto &&
|
||||
other.isActive == isActive &&
|
||||
other.queueCount == queueCount;
|
||||
|
||||
@override
|
||||
int get hashCode =>
|
||||
// ignore: unnecessary_parenthesis
|
||||
(isActive.hashCode) +
|
||||
(queueCount.hashCode);
|
||||
|
||||
@override
|
||||
String toString() => 'JobStatusResponseDto[isActive=$isActive, queueCount=$queueCount]';
|
||||
|
||||
Map<String, dynamic> toJson() {
|
||||
final json = <String, dynamic>{};
|
||||
json[r'isActive'] = this.isActive;
|
||||
json[r'queueCount'] = this.queueCount;
|
||||
return json;
|
||||
}
|
||||
|
||||
/// Returns a new [JobStatusResponseDto] instance and imports its values from
|
||||
/// [value] if it's a [Map], null otherwise.
|
||||
// ignore: prefer_constructors_over_static_methods
|
||||
static JobStatusResponseDto? fromJson(dynamic value) {
|
||||
if (value is Map) {
|
||||
final json = value.cast<String, dynamic>();
|
||||
|
||||
// Ensure that the map contains the required keys.
|
||||
// Note 1: the values aren't checked for validity beyond being non-null.
|
||||
// Note 2: this code is stripped in release mode!
|
||||
assert(() {
|
||||
requiredKeys.forEach((key) {
|
||||
assert(json.containsKey(key), 'Required key "JobStatusResponseDto[$key]" is missing from JSON.');
|
||||
assert(json[key] != null, 'Required key "JobStatusResponseDto[$key]" has a null value in JSON.');
|
||||
});
|
||||
return true;
|
||||
}());
|
||||
|
||||
return JobStatusResponseDto(
|
||||
isActive: mapValueOfType<bool>(json, r'isActive')!,
|
||||
queueCount: mapValueOfType<Object>(json, r'queueCount')!,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static List<JobStatusResponseDto>? listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <JobStatusResponseDto>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = JobStatusResponseDto.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
|
||||
static Map<String, JobStatusResponseDto> mapFromJson(dynamic json) {
|
||||
final map = <String, JobStatusResponseDto>{};
|
||||
if (json is Map && json.isNotEmpty) {
|
||||
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
|
||||
for (final entry in json.entries) {
|
||||
final value = JobStatusResponseDto.fromJson(entry.value);
|
||||
if (value != null) {
|
||||
map[entry.key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// maps a json object with a list of JobStatusResponseDto-objects as value to a dart map
|
||||
static Map<String, List<JobStatusResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
|
||||
final map = <String, List<JobStatusResponseDto>>{};
|
||||
if (json is Map && json.isNotEmpty) {
|
||||
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
|
||||
for (final entry in json.entries) {
|
||||
final value = JobStatusResponseDto.listFromJson(entry.value, growable: growable,);
|
||||
if (value != null) {
|
||||
map[entry.key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/// The list of required keys that must be present in a JSON.
|
||||
static const requiredKeys = <String>{
|
||||
'isActive',
|
||||
'queueCount',
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
import 'package:openapi/api.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
// tests for JobStatusResponseDto
|
||||
void main() {
|
||||
// final instance = JobStatusResponseDto();
|
||||
|
||||
group('test JobStatusResponseDto', () {
|
||||
// bool isActive
|
||||
test('to test the property `isActive`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// Object queueCount
|
||||
test('to test the property `queueCount`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
@ -1,217 +1,118 @@
|
||||
import {
|
||||
IMachineLearningJob,
|
||||
IMetadataExtractionJob,
|
||||
IThumbnailGenerationJob,
|
||||
IVideoTranscodeJob,
|
||||
JobName,
|
||||
QueueName,
|
||||
} from '@app/domain';
|
||||
import { InjectQueue } from '@nestjs/bull';
|
||||
import { Queue } from 'bull';
|
||||
import { JobName, IJobRepository, QueueName } from '@app/domain';
|
||||
import { BadRequestException, Inject, Injectable } from '@nestjs/common';
|
||||
import { AllJobStatusResponseDto } from './response-dto/all-job-status-response.dto';
|
||||
import { IAssetRepository } from '../asset/asset-repository';
|
||||
import { AssetType } from '@app/infra';
|
||||
import { GetJobDto, JobId } from './dto/get-job.dto';
|
||||
import { JobStatusResponseDto } from './response-dto/job-status-response.dto';
|
||||
import { StorageService } from '@app/storage';
|
||||
import { JobId } from './dto/get-job.dto';
|
||||
import { MACHINE_LEARNING_ENABLED } from '@app/common';
|
||||
|
||||
const jobIds = Object.values(JobId) as JobId[];
|
||||
|
||||
@Injectable()
|
||||
export class JobService {
|
||||
constructor(
|
||||
@InjectQueue(QueueName.THUMBNAIL_GENERATION)
|
||||
private thumbnailGeneratorQueue: Queue<IThumbnailGenerationJob>,
|
||||
|
||||
@InjectQueue(QueueName.METADATA_EXTRACTION)
|
||||
private metadataExtractionQueue: Queue<IMetadataExtractionJob>,
|
||||
|
||||
@InjectQueue(QueueName.VIDEO_CONVERSION)
|
||||
private videoConversionQueue: Queue<IVideoTranscodeJob>,
|
||||
|
||||
@InjectQueue(QueueName.MACHINE_LEARNING)
|
||||
private machineLearningQueue: Queue<IMachineLearningJob>,
|
||||
|
||||
@InjectQueue(QueueName.CONFIG)
|
||||
private configQueue: Queue,
|
||||
|
||||
@Inject(IAssetRepository)
|
||||
private _assetRepository: IAssetRepository,
|
||||
|
||||
private storageService: StorageService,
|
||||
@Inject(IAssetRepository) private _assetRepository: IAssetRepository,
|
||||
@Inject(IJobRepository) private jobRepository: IJobRepository,
|
||||
) {
|
||||
this.thumbnailGeneratorQueue.empty();
|
||||
this.metadataExtractionQueue.empty();
|
||||
this.videoConversionQueue.empty();
|
||||
this.configQueue.empty();
|
||||
}
|
||||
|
||||
async startJob(jobDto: GetJobDto): Promise<number> {
|
||||
switch (jobDto.jobId) {
|
||||
case JobId.THUMBNAIL_GENERATION:
|
||||
return this.runThumbnailGenerationJob();
|
||||
case JobId.METADATA_EXTRACTION:
|
||||
return this.runMetadataExtractionJob();
|
||||
case JobId.VIDEO_CONVERSION:
|
||||
return this.runVideoConversionJob();
|
||||
case JobId.MACHINE_LEARNING:
|
||||
return this.runMachineLearningPipeline();
|
||||
case JobId.STORAGE_TEMPLATE_MIGRATION:
|
||||
return this.runStorageMigration();
|
||||
default:
|
||||
throw new BadRequestException('Invalid job id');
|
||||
for (const jobId of jobIds) {
|
||||
this.jobRepository.empty(this.asQueueName(jobId));
|
||||
}
|
||||
}
|
||||
|
||||
async getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
|
||||
const thumbnailGeneratorJobCount = await this.thumbnailGeneratorQueue.getJobCounts();
|
||||
const metadataExtractionJobCount = await this.metadataExtractionQueue.getJobCounts();
|
||||
const videoConversionJobCount = await this.videoConversionQueue.getJobCounts();
|
||||
const machineLearningJobCount = await this.machineLearningQueue.getJobCounts();
|
||||
const storageMigrationJobCount = await this.configQueue.getJobCounts();
|
||||
|
||||
const response = new AllJobStatusResponseDto();
|
||||
response.isThumbnailGenerationActive = Boolean(thumbnailGeneratorJobCount.waiting);
|
||||
response.thumbnailGenerationQueueCount = thumbnailGeneratorJobCount;
|
||||
response.isMetadataExtractionActive = Boolean(metadataExtractionJobCount.waiting);
|
||||
response.metadataExtractionQueueCount = metadataExtractionJobCount;
|
||||
response.isVideoConversionActive = Boolean(videoConversionJobCount.waiting);
|
||||
response.videoConversionQueueCount = videoConversionJobCount;
|
||||
response.isMachineLearningActive = Boolean(machineLearningJobCount.waiting);
|
||||
response.machineLearningQueueCount = machineLearningJobCount;
|
||||
response.isStorageMigrationActive = Boolean(storageMigrationJobCount.active);
|
||||
response.storageMigrationQueueCount = storageMigrationJobCount;
|
||||
|
||||
return response;
|
||||
start(jobId: JobId): Promise<number> {
|
||||
return this.run(this.asQueueName(jobId));
|
||||
}
|
||||
|
||||
async getJobStatus(query: GetJobDto): Promise<JobStatusResponseDto> {
|
||||
const response = new JobStatusResponseDto();
|
||||
if (query.jobId === JobId.THUMBNAIL_GENERATION) {
|
||||
response.isActive = Boolean((await this.thumbnailGeneratorQueue.getJobCounts()).waiting);
|
||||
response.queueCount = await this.thumbnailGeneratorQueue.getJobCounts();
|
||||
}
|
||||
|
||||
if (query.jobId === JobId.METADATA_EXTRACTION) {
|
||||
response.isActive = Boolean((await this.metadataExtractionQueue.getJobCounts()).waiting);
|
||||
response.queueCount = await this.metadataExtractionQueue.getJobCounts();
|
||||
}
|
||||
|
||||
if (query.jobId === JobId.VIDEO_CONVERSION) {
|
||||
response.isActive = Boolean((await this.videoConversionQueue.getJobCounts()).waiting);
|
||||
response.queueCount = await this.videoConversionQueue.getJobCounts();
|
||||
}
|
||||
|
||||
if (query.jobId === JobId.STORAGE_TEMPLATE_MIGRATION) {
|
||||
response.isActive = Boolean((await this.configQueue.getJobCounts()).waiting);
|
||||
response.queueCount = await this.configQueue.getJobCounts();
|
||||
}
|
||||
|
||||
return response;
|
||||
async stop(jobId: JobId): Promise<number> {
|
||||
await this.jobRepository.empty(this.asQueueName(jobId));
|
||||
return 0;
|
||||
}
|
||||
|
||||
async stopJob(query: GetJobDto): Promise<number> {
|
||||
switch (query.jobId) {
|
||||
case JobId.THUMBNAIL_GENERATION:
|
||||
this.thumbnailGeneratorQueue.empty();
|
||||
return 0;
|
||||
case JobId.METADATA_EXTRACTION:
|
||||
this.metadataExtractionQueue.empty();
|
||||
return 0;
|
||||
case JobId.VIDEO_CONVERSION:
|
||||
this.videoConversionQueue.empty();
|
||||
return 0;
|
||||
case JobId.MACHINE_LEARNING:
|
||||
this.machineLearningQueue.empty();
|
||||
return 0;
|
||||
case JobId.STORAGE_TEMPLATE_MIGRATION:
|
||||
this.configQueue.empty();
|
||||
return 0;
|
||||
default:
|
||||
throw new BadRequestException('Invalid job id');
|
||||
async getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
|
||||
const response = new AllJobStatusResponseDto();
|
||||
for (const jobId of jobIds) {
|
||||
response[jobId] = await this.jobRepository.getJobCounts(this.asQueueName(jobId));
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
private async runThumbnailGenerationJob(): Promise<number> {
|
||||
const jobCount = await this.thumbnailGeneratorQueue.getJobCounts();
|
||||
|
||||
if (jobCount.waiting > 0) {
|
||||
throw new BadRequestException('Thumbnail generation job is already running');
|
||||
private async run(name: QueueName): Promise<number> {
|
||||
const isActive = await this.jobRepository.isActive(name);
|
||||
if (isActive) {
|
||||
throw new BadRequestException(`Job is already running`);
|
||||
}
|
||||
|
||||
const assetsWithNoThumbnail = await this._assetRepository.getAssetWithNoThumbnail();
|
||||
switch (name) {
|
||||
case QueueName.VIDEO_CONVERSION: {
|
||||
const assets = await this._assetRepository.getAssetWithNoEncodedVideo();
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.add({ name: JobName.VIDEO_CONVERSION, data: { asset } });
|
||||
}
|
||||
|
||||
for (const asset of assetsWithNoThumbnail) {
|
||||
await this.thumbnailGeneratorQueue.add(JobName.GENERATE_JPEG_THUMBNAIL, { asset });
|
||||
}
|
||||
|
||||
return assetsWithNoThumbnail.length;
|
||||
}
|
||||
|
||||
private async runMetadataExtractionJob(): Promise<number> {
|
||||
const jobCount = await this.metadataExtractionQueue.getJobCounts();
|
||||
|
||||
if (jobCount.waiting > 0) {
|
||||
throw new BadRequestException('Metadata extraction job is already running');
|
||||
}
|
||||
|
||||
const assetsWithNoExif = await this._assetRepository.getAssetWithNoEXIF();
|
||||
for (const asset of assetsWithNoExif) {
|
||||
if (asset.type === AssetType.VIDEO) {
|
||||
await this.metadataExtractionQueue.add(JobName.EXTRACT_VIDEO_METADATA, { asset, fileName: asset.id });
|
||||
} else {
|
||||
await this.metadataExtractionQueue.add(JobName.EXIF_EXTRACTION, { asset, fileName: asset.id });
|
||||
return assets.length;
|
||||
}
|
||||
}
|
||||
return assetsWithNoExif.length;
|
||||
}
|
||||
|
||||
private async runMachineLearningPipeline(): Promise<number> {
|
||||
if (!MACHINE_LEARNING_ENABLED) {
|
||||
throw new BadRequestException('Machine learning is not enabled.');
|
||||
}
|
||||
|
||||
const jobCount = await this.machineLearningQueue.getJobCounts();
|
||||
case QueueName.CONFIG:
|
||||
await this.jobRepository.add({ name: JobName.TEMPLATE_MIGRATION });
|
||||
return 1;
|
||||
|
||||
case QueueName.MACHINE_LEARNING: {
|
||||
if (!MACHINE_LEARNING_ENABLED) {
|
||||
throw new BadRequestException('Machine learning is not enabled.');
|
||||
}
|
||||
|
||||
const assets = await this._assetRepository.getAssetWithNoSmartInfo();
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.add({ name: JobName.IMAGE_TAGGING, data: { asset } });
|
||||
await this.jobRepository.add({ name: JobName.OBJECT_DETECTION, data: { asset } });
|
||||
}
|
||||
return assets.length;
|
||||
}
|
||||
|
||||
if (jobCount.waiting > 0) {
|
||||
throw new BadRequestException('Metadata extraction job is already running');
|
||||
}
|
||||
case QueueName.METADATA_EXTRACTION: {
|
||||
const assets = await this._assetRepository.getAssetWithNoEXIF();
|
||||
for (const asset of assets) {
|
||||
if (asset.type === AssetType.VIDEO) {
|
||||
await this.jobRepository.add({ name: JobName.EXTRACT_VIDEO_METADATA, data: { asset, fileName: asset.id } });
|
||||
} else {
|
||||
await this.jobRepository.add({ name: JobName.EXIF_EXTRACTION, data: { asset, fileName: asset.id } });
|
||||
}
|
||||
}
|
||||
return assets.length;
|
||||
}
|
||||
|
||||
const assetWithNoSmartInfo = await this._assetRepository.getAssetWithNoSmartInfo();
|
||||
case QueueName.THUMBNAIL_GENERATION: {
|
||||
const assets = await this._assetRepository.getAssetWithNoThumbnail();
|
||||
for (const asset of assets) {
|
||||
await this.jobRepository.add({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { asset } });
|
||||
}
|
||||
return assets.length;
|
||||
}
|
||||
|
||||
for (const asset of assetWithNoSmartInfo) {
|
||||
await this.machineLearningQueue.add(JobName.IMAGE_TAGGING, { asset });
|
||||
await this.machineLearningQueue.add(JobName.OBJECT_DETECTION, { asset });
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
|
||||
return assetWithNoSmartInfo.length;
|
||||
}
|
||||
|
||||
private async runVideoConversionJob(): Promise<number> {
|
||||
const jobCount = await this.videoConversionQueue.getJobCounts();
|
||||
|
||||
if (jobCount.waiting > 0) {
|
||||
throw new BadRequestException('Video conversion job is already running');
|
||||
}
|
||||
private asQueueName(jobId: JobId) {
|
||||
switch (jobId) {
|
||||
case JobId.THUMBNAIL_GENERATION:
|
||||
return QueueName.THUMBNAIL_GENERATION;
|
||||
|
||||
const assetsWithNoConvertedVideo = await this._assetRepository.getAssetWithNoEncodedVideo();
|
||||
case JobId.METADATA_EXTRACTION:
|
||||
return QueueName.METADATA_EXTRACTION;
|
||||
|
||||
for (const asset of assetsWithNoConvertedVideo) {
|
||||
await this.videoConversionQueue.add(JobName.VIDEO_CONVERSION, { asset });
|
||||
}
|
||||
case JobId.VIDEO_CONVERSION:
|
||||
return QueueName.VIDEO_CONVERSION;
|
||||
|
||||
return assetsWithNoConvertedVideo.length;
|
||||
}
|
||||
case JobId.STORAGE_TEMPLATE_MIGRATION:
|
||||
return QueueName.CONFIG;
|
||||
|
||||
async runStorageMigration() {
|
||||
const jobCount = await this.configQueue.getJobCounts();
|
||||
case JobId.MACHINE_LEARNING:
|
||||
return QueueName.MACHINE_LEARNING;
|
||||
|
||||
if (jobCount.active > 0) {
|
||||
throw new BadRequestException('Storage migration job is already running');
|
||||
default:
|
||||
throw new BadRequestException(`Invalid job id: ${jobId}`);
|
||||
}
|
||||
|
||||
await this.configQueue.add(JobName.TEMPLATE_MIGRATION, {});
|
||||
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +0,0 @@
|
||||
import Bull from 'bull';
|
||||
|
||||
export class JobStatusResponseDto {
|
||||
isActive!: boolean;
|
||||
queueCount!: Bull.JobCounts;
|
||||
}
|
||||
@ -1,12 +1,9 @@
|
||||
import { BullModule } from '@nestjs/bull';
|
||||
import { Module } from '@nestjs/common';
|
||||
import { QueueName } from '@app/domain';
|
||||
import { BackgroundTaskProcessor } from './background-task.processor';
|
||||
import { BackgroundTaskService } from './background-task.service';
|
||||
|
||||
@Module({
|
||||
imports: [BullModule.registerQueue({ name: QueueName.BACKGROUND_TASK })],
|
||||
providers: [BackgroundTaskService, BackgroundTaskProcessor],
|
||||
exports: [BackgroundTaskService, BullModule],
|
||||
exports: [BackgroundTaskService],
|
||||
})
|
||||
export class BackgroundTaskModule {}
|
||||
|
||||
@ -1,17 +1,12 @@
|
||||
import { InjectQueue } from '@nestjs/bull/dist/decorators';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Queue } from 'bull';
|
||||
import { JobName, QueueName } from '@app/domain';
|
||||
import { AssetResponseDto } from '../../api-v1/asset/response-dto/asset-response.dto';
|
||||
import { IJobRepository, JobName } from '@app/domain';
|
||||
import { AssetEntity } from '@app/infra';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
|
||||
@Injectable()
|
||||
export class BackgroundTaskService {
|
||||
constructor(
|
||||
@InjectQueue(QueueName.BACKGROUND_TASK)
|
||||
private backgroundTaskQueue: Queue,
|
||||
) {}
|
||||
constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {}
|
||||
|
||||
async deleteFileOnDisk(assets: AssetResponseDto[]) {
|
||||
await this.backgroundTaskQueue.add(JobName.DELETE_FILE_ON_DISK, { assets });
|
||||
async deleteFileOnDisk(assets: AssetEntity[]) {
|
||||
await this.jobRepository.add({ name: JobName.DELETE_FILE_ON_DISK, data: { assets } });
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,17 +1,16 @@
|
||||
import { QueueName } from '@app/domain';
|
||||
import { InjectQueue } from '@nestjs/bull';
|
||||
import { Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { Queue } from 'bull';
|
||||
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { IJobRepository, JobName } from '@app/domain';
|
||||
|
||||
const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(() => resolve(), ms));
|
||||
|
||||
@Injectable()
|
||||
export class MicroservicesService implements OnModuleInit {
|
||||
constructor(
|
||||
@InjectQueue(QueueName.CHECKSUM_GENERATION)
|
||||
private generateChecksumQueue: Queue,
|
||||
) {}
|
||||
constructor(@Inject(IJobRepository) private jobRepository: IJobRepository) {}
|
||||
|
||||
async onModuleInit() {
|
||||
// wait for migration
|
||||
await this.generateChecksumQueue.add({}, { delay: 10000 });
|
||||
await sleep(10_000);
|
||||
|
||||
await this.jobRepository.add({ name: JobName.CHECKSUM_GENERATION });
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,21 +1,110 @@
|
||||
import { IJobRepository, JobItem, JobName, QueueName } from '@app/domain';
|
||||
import {
|
||||
IAssetUploadedJob,
|
||||
IJobRepository,
|
||||
IMachineLearningJob,
|
||||
IMetadataExtractionJob,
|
||||
IUserDeletionJob,
|
||||
IVideoTranscodeJob,
|
||||
JobCounts,
|
||||
JobItem,
|
||||
JobName,
|
||||
QueueName,
|
||||
} from '@app/domain';
|
||||
import { InjectQueue } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { BadRequestException, Logger } from '@nestjs/common';
|
||||
import { Queue } from 'bull';
|
||||
|
||||
export class JobRepository implements IJobRepository {
|
||||
private logger = new Logger(JobRepository.name);
|
||||
|
||||
constructor(@InjectQueue(QueueName.CONFIG) private configQueue: Queue) {}
|
||||
constructor(
|
||||
@InjectQueue(QueueName.ASSET_UPLOADED) private assetUploaded: Queue<IAssetUploadedJob>,
|
||||
@InjectQueue(QueueName.BACKGROUND_TASK) private backgroundTask: Queue,
|
||||
@InjectQueue(QueueName.CHECKSUM_GENERATION) private generateChecksum: Queue,
|
||||
@InjectQueue(QueueName.MACHINE_LEARNING) private machineLearning: Queue<IMachineLearningJob>,
|
||||
@InjectQueue(QueueName.METADATA_EXTRACTION) private metadataExtraction: Queue<IMetadataExtractionJob>,
|
||||
@InjectQueue(QueueName.CONFIG) private storageMigration: Queue,
|
||||
@InjectQueue(QueueName.THUMBNAIL_GENERATION) private thumbnail: Queue,
|
||||
@InjectQueue(QueueName.USER_DELETION) private userDeletion: Queue<IUserDeletionJob>,
|
||||
@InjectQueue(QueueName.VIDEO_CONVERSION) private videoTranscode: Queue<IVideoTranscodeJob>,
|
||||
) {}
|
||||
|
||||
async isActive(name: QueueName): Promise<boolean> {
|
||||
const counts = await this.getJobCounts(name);
|
||||
return !!counts.active;
|
||||
}
|
||||
|
||||
empty(name: QueueName) {
|
||||
return this.getQueue(name).empty();
|
||||
}
|
||||
|
||||
getJobCounts(name: QueueName): Promise<JobCounts> {
|
||||
return this.getQueue(name).getJobCounts();
|
||||
}
|
||||
|
||||
async add(item: JobItem): Promise<void> {
|
||||
switch (item.name) {
|
||||
case JobName.ASSET_UPLOADED:
|
||||
await this.assetUploaded.add(item.name, item.data, { jobId: item.data.asset.id });
|
||||
break;
|
||||
|
||||
case JobName.DELETE_FILE_ON_DISK:
|
||||
await this.backgroundTask.add(item.name, item.data);
|
||||
break;
|
||||
|
||||
case JobName.CHECKSUM_GENERATION:
|
||||
await this.generateChecksum.add(item.name, {});
|
||||
break;
|
||||
|
||||
case JobName.OBJECT_DETECTION:
|
||||
case JobName.IMAGE_TAGGING:
|
||||
await this.machineLearning.add(item.name, item.data);
|
||||
break;
|
||||
|
||||
case JobName.EXIF_EXTRACTION:
|
||||
case JobName.EXTRACT_VIDEO_METADATA:
|
||||
case JobName.REVERSE_GEOCODING:
|
||||
await this.metadataExtraction.add(item.name, item.data);
|
||||
break;
|
||||
|
||||
case JobName.TEMPLATE_MIGRATION:
|
||||
case JobName.CONFIG_CHANGE:
|
||||
await this.configQueue.add(JobName.CONFIG_CHANGE, {});
|
||||
await this.storageMigration.add(item.name, {});
|
||||
break;
|
||||
|
||||
case JobName.GENERATE_JPEG_THUMBNAIL:
|
||||
case JobName.GENERATE_WEBP_THUMBNAIL:
|
||||
await this.thumbnail.add(item.name, item.data);
|
||||
break;
|
||||
|
||||
case JobName.USER_DELETION:
|
||||
await this.userDeletion.add(item.name, item.data);
|
||||
break;
|
||||
|
||||
case JobName.VIDEO_CONVERSION:
|
||||
await this.videoTranscode.add(item.name, item.data);
|
||||
break;
|
||||
|
||||
default:
|
||||
// TODO inject remaining queues and map job to queue
|
||||
this.logger.error('Invalid job', item);
|
||||
}
|
||||
}
|
||||
|
||||
private getQueue(name: QueueName) {
|
||||
switch (name) {
|
||||
case QueueName.THUMBNAIL_GENERATION:
|
||||
return this.thumbnail;
|
||||
case QueueName.METADATA_EXTRACTION:
|
||||
return this.metadataExtraction;
|
||||
case QueueName.VIDEO_CONVERSION:
|
||||
return this.videoTranscode;
|
||||
case QueueName.CONFIG:
|
||||
return this.storageMigration;
|
||||
case QueueName.MACHINE_LEARNING:
|
||||
return this.machineLearning;
|
||||
default:
|
||||
throw new BadRequestException('Invalid job name');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue