mirror of https://github.com/immich-app/immich.git
chore(server) Add job for storage migration (#1117)
Alex
GitHub
parent
8998a79ff9
commit
de69d0031e
@ -0,0 +1,61 @@
|
||||
import { APP_UPLOAD_LOCATION } from '@app/common';
|
||||
import { AssetEntity } from '@app/database/entities/asset.entity';
|
||||
import { ImmichConfigService } from '@app/immich-config';
|
||||
import { QueueNameEnum, templateMigrationProcessorName, updateTemplateProcessorName } from '@app/job';
|
||||
import { StorageService } from '@app/storage';
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
@Processor(QueueNameEnum.STORAGE_MIGRATION)
|
||||
export class StorageMigrationProcessor {
|
||||
readonly logger: Logger = new Logger(StorageMigrationProcessor.name);
|
||||
|
||||
constructor(
|
||||
private storageService: StorageService,
|
||||
private immichConfigService: ImmichConfigService,
|
||||
|
||||
@InjectRepository(AssetEntity)
|
||||
private assetRepository: Repository<AssetEntity>,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Migration process when a new user set a new storage template.
|
||||
* @param job
|
||||
*/
|
||||
@Process({ name: templateMigrationProcessorName, concurrency: 100 })
|
||||
async templateMigration() {
|
||||
console.time('migrating-time');
|
||||
const assets = await this.assetRepository.find({
|
||||
relations: ['exifInfo'],
|
||||
});
|
||||
|
||||
const livePhotoMap: Record<string, AssetEntity> = {};
|
||||
|
||||
for (const asset of assets) {
|
||||
if (asset.livePhotoVideoId) {
|
||||
livePhotoMap[asset.livePhotoVideoId] = asset;
|
||||
}
|
||||
}
|
||||
|
||||
for (const asset of assets) {
|
||||
const livePhotoParentAsset = livePhotoMap[asset.id];
|
||||
const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id;
|
||||
await this.storageService.moveAsset(asset, filename);
|
||||
}
|
||||
|
||||
await this.storageService.removeEmptyDirectories(APP_UPLOAD_LOCATION);
|
||||
console.timeEnd('migrating-time');
|
||||
}
|
||||
|
||||
/**
|
||||
* Update config when a new storage template is set.
|
||||
* This is to ensure the synchronization between processes.
|
||||
* @param job
|
||||
*/
|
||||
@Process({ name: updateTemplateProcessorName, concurrency: 1 })
|
||||
async updateTemplate() {
|
||||
await this.immichConfigService.refreshConfig();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
import { SharedBullAsyncConfiguration } from '@nestjs/bull';
|
||||
|
||||
export const immichBullAsyncConfig: SharedBullAsyncConfiguration = {
|
||||
useFactory: async () => ({
|
||||
prefix: 'immich_bull',
|
||||
redis: {
|
||||
host: process.env.REDIS_HOSTNAME || 'immich_redis',
|
||||
port: parseInt(process.env.REDIS_PORT || '6379'),
|
||||
db: parseInt(process.env.REDIS_DBINDEX || '0'),
|
||||
password: process.env.REDIS_PASSWORD || undefined,
|
||||
path: process.env.REDIS_SOCKET || undefined,
|
||||
},
|
||||
defaultJobOptions: {
|
||||
attempts: 3,
|
||||
removeOnComplete: true,
|
||||
removeOnFail: false,
|
||||
},
|
||||
}),
|
||||
};
|
||||
@ -1 +1,2 @@
|
||||
export * from './app.config';
|
||||
export * from './bull-queue.config';
|
||||
|
||||
@ -0,0 +1,32 @@
|
||||
import { BullModuleOptions } from '@nestjs/bull';
|
||||
import { QueueNameEnum } from './queue-name.constant';
|
||||
|
||||
/**
|
||||
* Shared queues between apps and microservices
|
||||
*/
|
||||
export const immichSharedQueues: BullModuleOptions[] = [
|
||||
{
|
||||
name: QueueNameEnum.USER_DELETION,
|
||||
},
|
||||
{
|
||||
name: QueueNameEnum.THUMBNAIL_GENERATION,
|
||||
},
|
||||
{
|
||||
name: QueueNameEnum.ASSET_UPLOADED,
|
||||
},
|
||||
{
|
||||
name: QueueNameEnum.METADATA_EXTRACTION,
|
||||
},
|
||||
{
|
||||
name: QueueNameEnum.VIDEO_CONVERSION,
|
||||
},
|
||||
{
|
||||
name: QueueNameEnum.CHECKSUM_GENERATION,
|
||||
},
|
||||
{
|
||||
name: QueueNameEnum.MACHINE_LEARNING,
|
||||
},
|
||||
{
|
||||
name: QueueNameEnum.STORAGE_MIGRATION,
|
||||
},
|
||||
];
|
||||
Loading…
Reference in New Issue