mirror of https://github.com/immich-app/immich.git
chore: organize config, validation, decorators (#8118)
* refactor: validation * refactor: utilities * refactor: configpull/8119/head
Jason Rasmussen
GitHub
parent
92cc647cf6
commit
81f0265095
@ -1,7 +1,42 @@
|
||||
import { RegisterQueueOptions } from '@nestjs/bullmq';
|
||||
import { ConfigModuleOptions } from '@nestjs/config';
|
||||
import { QueueOptions } from 'bullmq';
|
||||
import { RedisOptions } from 'ioredis';
|
||||
import Joi from 'joi';
|
||||
import { QueueName } from 'src/domain/job/job.constants';
|
||||
import { LogLevel } from 'src/infra/entities/system-config.entity';
|
||||
|
||||
const WHEN_DB_URL_SET = Joi.when('DB_URL', {
|
||||
is: Joi.exist(),
|
||||
then: Joi.string().optional(),
|
||||
otherwise: Joi.string().required(),
|
||||
});
|
||||
|
||||
export const immichAppConfig: ConfigModuleOptions = {
|
||||
envFilePath: '.env',
|
||||
isGlobal: true,
|
||||
validationSchema: Joi.object({
|
||||
NODE_ENV: Joi.string().optional().valid('development', 'production', 'staging').default('development'),
|
||||
LOG_LEVEL: Joi.string()
|
||||
.optional()
|
||||
.valid(...Object.values(LogLevel)),
|
||||
|
||||
DB_USERNAME: WHEN_DB_URL_SET,
|
||||
DB_PASSWORD: WHEN_DB_URL_SET,
|
||||
DB_DATABASE_NAME: WHEN_DB_URL_SET,
|
||||
DB_URL: Joi.string().optional(),
|
||||
DB_VECTOR_EXTENSION: Joi.string().optional().valid('pgvector', 'pgvecto.rs').default('pgvecto.rs'),
|
||||
|
||||
MACHINE_LEARNING_PORT: Joi.number().optional(),
|
||||
MICROSERVICES_PORT: Joi.number().optional(),
|
||||
IMMICH_METRICS_PORT: Joi.number().optional(),
|
||||
|
||||
IMMICH_METRICS: Joi.boolean().optional().default(false),
|
||||
IMMICH_HOST_METRICS: Joi.boolean().optional().default(false),
|
||||
IMMICH_API_METRICS: Joi.boolean().optional().default(false),
|
||||
IMMICH_IO_METRICS: Joi.boolean().optional().default(false),
|
||||
}),
|
||||
};
|
||||
|
||||
function parseRedisConfig(): RedisOptions {
|
||||
const redisUrl = process.env.REDIS_URL;
|
||||
@ -0,0 +1,124 @@
|
||||
import { SetMetadata } from '@nestjs/common';
|
||||
import _ from 'lodash';
|
||||
import { setUnion } from 'src/utils';
|
||||
|
||||
// PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the
|
||||
// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching
|
||||
// by a list of IDs) requires splitting the query into multiple chunks.
|
||||
// We are rounding down this limit, as queries commonly include other filters and parameters.
|
||||
export const DATABASE_PARAMETER_CHUNK_SIZE = 65_500;
|
||||
|
||||
/**
|
||||
* Chunks an array or set into smaller collections of the same type and specified size.
|
||||
*
|
||||
* @param collection The collection to chunk.
|
||||
* @param size The size of each chunk.
|
||||
*/
|
||||
function chunks<T>(collection: Array<T>, size: number): Array<Array<T>>;
|
||||
function chunks<T>(collection: Set<T>, size: number): Array<Set<T>>;
|
||||
function chunks<T>(collection: Array<T> | Set<T>, size: number): Array<Array<T>> | Array<Set<T>> {
|
||||
if (collection instanceof Set) {
|
||||
const result = [];
|
||||
let chunk = new Set<T>();
|
||||
for (const element of collection) {
|
||||
chunk.add(element);
|
||||
if (chunk.size === size) {
|
||||
result.push(chunk);
|
||||
chunk = new Set<T>();
|
||||
}
|
||||
}
|
||||
if (chunk.size > 0) {
|
||||
result.push(chunk);
|
||||
}
|
||||
return result;
|
||||
} else {
|
||||
return _.chunk(collection, size);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
|
||||
* to overcome the maximum number of parameters allowed by the database driver.
|
||||
*
|
||||
* @param options.paramIndex The index of the function parameter to chunk. Defaults to 0.
|
||||
* @param options.flatten Whether to flatten the results. Defaults to false.
|
||||
*/
|
||||
export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator {
|
||||
return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => {
|
||||
const originalMethod = descriptor.value;
|
||||
const parameterIndex = options.paramIndex ?? 0;
|
||||
descriptor.value = async function (...arguments_: any[]) {
|
||||
const argument = arguments_[parameterIndex];
|
||||
|
||||
// Early return if argument length is less than or equal to the chunk size.
|
||||
if (
|
||||
(Array.isArray(argument) && argument.length <= DATABASE_PARAMETER_CHUNK_SIZE) ||
|
||||
(argument instanceof Set && argument.size <= DATABASE_PARAMETER_CHUNK_SIZE)
|
||||
) {
|
||||
return await originalMethod.apply(this, arguments_);
|
||||
}
|
||||
|
||||
return Promise.all(
|
||||
chunks(argument, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => {
|
||||
await Reflect.apply(originalMethod, this, [
|
||||
...arguments_.slice(0, parameterIndex),
|
||||
chunk,
|
||||
...arguments_.slice(parameterIndex + 1),
|
||||
]);
|
||||
}),
|
||||
).then((results) => (options.mergeFn ? options.mergeFn(results) : results));
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator {
|
||||
return Chunked({ ...options, mergeFn: _.flatten });
|
||||
}
|
||||
|
||||
export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
|
||||
return Chunked({ ...options, mergeFn: setUnion });
|
||||
}
|
||||
|
||||
// https://stackoverflow.com/a/74898678
|
||||
export function DecorateAll(
|
||||
decorator: <T>(
|
||||
target: any,
|
||||
propertyKey: string,
|
||||
descriptor: TypedPropertyDescriptor<T>,
|
||||
) => TypedPropertyDescriptor<T> | void,
|
||||
) {
|
||||
return (target: any) => {
|
||||
const descriptors = Object.getOwnPropertyDescriptors(target.prototype);
|
||||
for (const [propName, descriptor] of Object.entries(descriptors)) {
|
||||
const isMethod = typeof descriptor.value == 'function' && propName !== 'constructor';
|
||||
if (!isMethod) {
|
||||
continue;
|
||||
}
|
||||
decorator({ ...target, constructor: { ...target.constructor, name: target.name } as any }, propName, descriptor);
|
||||
Object.defineProperty(target.prototype, propName, descriptor);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const UUID = '00000000-0000-4000-a000-000000000000';
|
||||
|
||||
export const DummyValue = {
|
||||
UUID,
|
||||
UUID_SET: new Set([UUID]),
|
||||
PAGINATION: { take: 10, skip: 0 },
|
||||
EMAIL: 'user@immich.app',
|
||||
STRING: 'abcdefghi',
|
||||
BUFFER: Buffer.from('abcdefghi'),
|
||||
DATE: new Date(),
|
||||
TIME_BUCKET: '2024-01-01T00:00:00.000Z',
|
||||
};
|
||||
|
||||
export const GENERATE_SQL_KEY = 'generate-sql-key';
|
||||
|
||||
export interface GenerateSqlQueries {
|
||||
name?: string;
|
||||
params: unknown[];
|
||||
}
|
||||
|
||||
/** Decorator to enable versioning/tracking of generated Sql */
|
||||
export const GenerateSql = (...options: GenerateSqlQueries[]) => SetMetadata(GENERATE_SQL_KEY, options);
|
||||
@ -1,36 +0,0 @@
|
||||
// TODO: remove nestjs references from domain
|
||||
import { ConfigModuleOptions } from '@nestjs/config';
|
||||
import Joi from 'joi';
|
||||
import { LogLevel } from 'src/infra/entities/system-config.entity';
|
||||
|
||||
const WHEN_DB_URL_SET = Joi.when('DB_URL', {
|
||||
is: Joi.exist(),
|
||||
then: Joi.string().optional(),
|
||||
otherwise: Joi.string().required(),
|
||||
});
|
||||
|
||||
export const immichAppConfig: ConfigModuleOptions = {
|
||||
envFilePath: '.env',
|
||||
isGlobal: true,
|
||||
validationSchema: Joi.object({
|
||||
NODE_ENV: Joi.string().optional().valid('development', 'production', 'staging').default('development'),
|
||||
LOG_LEVEL: Joi.string()
|
||||
.optional()
|
||||
.valid(...Object.values(LogLevel)),
|
||||
|
||||
DB_USERNAME: WHEN_DB_URL_SET,
|
||||
DB_PASSWORD: WHEN_DB_URL_SET,
|
||||
DB_DATABASE_NAME: WHEN_DB_URL_SET,
|
||||
DB_URL: Joi.string().optional(),
|
||||
DB_VECTOR_EXTENSION: Joi.string().optional().valid('pgvector', 'pgvecto.rs').default('pgvecto.rs'),
|
||||
|
||||
MACHINE_LEARNING_PORT: Joi.number().optional(),
|
||||
MICROSERVICES_PORT: Joi.number().optional(),
|
||||
IMMICH_METRICS_PORT: Joi.number().optional(),
|
||||
|
||||
IMMICH_METRICS: Joi.boolean().optional().default(false),
|
||||
IMMICH_HOST_METRICS: Joi.boolean().optional().default(false),
|
||||
IMMICH_API_METRICS: Joi.boolean().optional().default(false),
|
||||
IMMICH_IO_METRICS: Joi.boolean().optional().default(false),
|
||||
}),
|
||||
};
|
||||
@ -1,280 +0,0 @@
|
||||
import { BadRequestException, applyDecorators } from '@nestjs/common';
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { Transform } from 'class-transformer';
|
||||
import {
|
||||
IsArray,
|
||||
IsBoolean,
|
||||
IsDate,
|
||||
IsNotEmpty,
|
||||
IsOptional,
|
||||
IsString,
|
||||
IsUUID,
|
||||
ValidateIf,
|
||||
ValidationOptions,
|
||||
isDateString,
|
||||
} from 'class-validator';
|
||||
import { CronJob } from 'cron';
|
||||
import _ from 'lodash';
|
||||
import { basename, extname } from 'node:path';
|
||||
import sanitize from 'sanitize-filename';
|
||||
import { ImmichLogger } from 'src/infra/logger';
|
||||
|
||||
export enum CacheControl {
|
||||
PRIVATE_WITH_CACHE = 'private_with_cache',
|
||||
PRIVATE_WITHOUT_CACHE = 'private_without_cache',
|
||||
NONE = 'none',
|
||||
}
|
||||
|
||||
export class ImmichFileResponse {
|
||||
public readonly path!: string;
|
||||
public readonly contentType!: string;
|
||||
public readonly cacheControl!: CacheControl;
|
||||
|
||||
constructor(response: ImmichFileResponse) {
|
||||
Object.assign(this, response);
|
||||
}
|
||||
}
|
||||
|
||||
export interface OpenGraphTags {
|
||||
title: string;
|
||||
description: string;
|
||||
imageUrl?: string;
|
||||
}
|
||||
|
||||
export const isConnectionAborted = (error: Error | any) => error.code === 'ECONNABORTED';
|
||||
|
||||
type UUIDOptions = { optional?: boolean; each?: boolean };
|
||||
export const ValidateUUID = (options?: UUIDOptions) => {
|
||||
const { optional, each } = { optional: false, each: false, ...options };
|
||||
return applyDecorators(
|
||||
IsUUID('4', { each }),
|
||||
ApiProperty({ format: 'uuid' }),
|
||||
optional ? Optional() : IsNotEmpty(),
|
||||
each ? IsArray() : IsString(),
|
||||
);
|
||||
};
|
||||
|
||||
type DateOptions = { optional?: boolean; nullable?: boolean; format?: 'date' | 'date-time' };
|
||||
export const ValidateDate = (options?: DateOptions) => {
|
||||
const { optional, nullable, format } = { optional: false, nullable: false, format: 'date-time', ...options };
|
||||
|
||||
const decorators = [
|
||||
ApiProperty({ format }),
|
||||
IsDate(),
|
||||
optional ? Optional({ nullable: true }) : IsNotEmpty(),
|
||||
Transform(({ key, value }) => {
|
||||
if (value === null || value === undefined) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (!isDateString(value)) {
|
||||
throw new BadRequestException(`${key} must be a date string`);
|
||||
}
|
||||
|
||||
return new Date(value as string);
|
||||
}),
|
||||
];
|
||||
|
||||
if (optional) {
|
||||
decorators.push(Optional({ nullable }));
|
||||
}
|
||||
|
||||
return applyDecorators(...decorators);
|
||||
};
|
||||
|
||||
type BooleanOptions = { optional?: boolean };
|
||||
export const ValidateBoolean = (options?: BooleanOptions) => {
|
||||
const { optional } = { optional: false, ...options };
|
||||
const decorators = [
|
||||
// ApiProperty(),
|
||||
IsBoolean(),
|
||||
Transform(({ value }) => {
|
||||
if (value == 'true') {
|
||||
return true;
|
||||
} else if (value == 'false') {
|
||||
return false;
|
||||
}
|
||||
return value;
|
||||
}),
|
||||
];
|
||||
|
||||
if (optional) {
|
||||
decorators.push(Optional());
|
||||
}
|
||||
|
||||
return applyDecorators(...decorators);
|
||||
};
|
||||
|
||||
export function validateCronExpression(expression: string) {
|
||||
try {
|
||||
new CronJob(expression, () => {});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
type IValue = { value: string };
|
||||
|
||||
export const toEmail = ({ value }: IValue) => value?.toLowerCase();
|
||||
|
||||
export const toSanitized = ({ value }: IValue) => sanitize((value || '').replaceAll('.', ''));
|
||||
|
||||
export function getFileNameWithoutExtension(path: string): string {
|
||||
return basename(path, extname(path));
|
||||
}
|
||||
|
||||
export function getLivePhotoMotionFilename(stillName: string, motionName: string) {
|
||||
return getFileNameWithoutExtension(stillName) + extname(motionName);
|
||||
}
|
||||
|
||||
const KiB = Math.pow(1024, 1);
|
||||
const MiB = Math.pow(1024, 2);
|
||||
const GiB = Math.pow(1024, 3);
|
||||
const TiB = Math.pow(1024, 4);
|
||||
const PiB = Math.pow(1024, 5);
|
||||
|
||||
export const HumanReadableSize = { KiB, MiB, GiB, TiB, PiB };
|
||||
|
||||
export function asHumanReadable(bytes: number, precision = 1): string {
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB'];
|
||||
|
||||
let magnitude = 0;
|
||||
let remainder = bytes;
|
||||
while (remainder >= 1024) {
|
||||
if (magnitude + 1 < units.length) {
|
||||
magnitude++;
|
||||
remainder /= 1024;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return `${remainder.toFixed(magnitude == 0 ? 0 : precision)} ${units[magnitude]}`;
|
||||
}
|
||||
|
||||
export interface PaginationOptions {
|
||||
take: number;
|
||||
skip?: number;
|
||||
}
|
||||
|
||||
export enum PaginationMode {
|
||||
LIMIT_OFFSET = 'limit-offset',
|
||||
SKIP_TAKE = 'skip-take',
|
||||
}
|
||||
|
||||
export interface PaginatedBuilderOptions {
|
||||
take: number;
|
||||
skip?: number;
|
||||
mode?: PaginationMode;
|
||||
}
|
||||
|
||||
export interface PaginationResult<T> {
|
||||
items: T[];
|
||||
hasNextPage: boolean;
|
||||
}
|
||||
|
||||
export type Paginated<T> = Promise<PaginationResult<T>>;
|
||||
|
||||
export async function* usePagination<T>(
|
||||
pageSize: number,
|
||||
getNextPage: (pagination: PaginationOptions) => PaginationResult<T> | Paginated<T>,
|
||||
) {
|
||||
let hasNextPage = true;
|
||||
|
||||
for (let skip = 0; hasNextPage; skip += pageSize) {
|
||||
const result = await getNextPage({ take: pageSize, skip });
|
||||
hasNextPage = result.hasNextPage;
|
||||
yield result.items;
|
||||
}
|
||||
}
|
||||
|
||||
export interface OptionalOptions extends ValidationOptions {
|
||||
nullable?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if value is missing and if so, ignores all validators.
|
||||
*
|
||||
* @param validationOptions {@link OptionalOptions}
|
||||
*
|
||||
* @see IsOptional exported from `class-validator.
|
||||
*/
|
||||
// https://stackoverflow.com/a/71353929
|
||||
export function Optional({ nullable, ...validationOptions }: OptionalOptions = {}) {
|
||||
if (nullable === true) {
|
||||
return IsOptional(validationOptions);
|
||||
}
|
||||
|
||||
return ValidateIf((object: any, v: any) => v !== undefined, validationOptions);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunks an array or set into smaller collections of the same type and specified size.
|
||||
*
|
||||
* @param collection The collection to chunk.
|
||||
* @param size The size of each chunk.
|
||||
*/
|
||||
export function chunks<T>(collection: Array<T>, size: number): Array<Array<T>>;
|
||||
export function chunks<T>(collection: Set<T>, size: number): Array<Set<T>>;
|
||||
export function chunks<T>(collection: Array<T> | Set<T>, size: number): Array<Array<T>> | Array<Set<T>> {
|
||||
if (collection instanceof Set) {
|
||||
const result = [];
|
||||
let chunk = new Set<T>();
|
||||
for (const element of collection) {
|
||||
chunk.add(element);
|
||||
if (chunk.size === size) {
|
||||
result.push(chunk);
|
||||
chunk = new Set<T>();
|
||||
}
|
||||
}
|
||||
if (chunk.size > 0) {
|
||||
result.push(chunk);
|
||||
}
|
||||
return result;
|
||||
} else {
|
||||
return _.chunk(collection, size);
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: The following Set utils have been added here, to easily determine where they are used.
|
||||
// They should be replaced with native Set operations, when they are added to the language.
|
||||
// Proposal reference: https://github.com/tc39/proposal-set-methods
|
||||
|
||||
export const setUnion = <T>(...sets: Set<T>[]): Set<T> => {
|
||||
const union = new Set(sets[0]);
|
||||
for (const set of sets.slice(1)) {
|
||||
for (const element of set) {
|
||||
union.add(element);
|
||||
}
|
||||
}
|
||||
return union;
|
||||
};
|
||||
|
||||
export const setDifference = <T>(setA: Set<T>, ...sets: Set<T>[]): Set<T> => {
|
||||
const difference = new Set(setA);
|
||||
for (const set of sets) {
|
||||
for (const element of set) {
|
||||
difference.delete(element);
|
||||
}
|
||||
}
|
||||
return difference;
|
||||
};
|
||||
|
||||
export const setIsSuperset = <T>(set: Set<T>, subset: Set<T>): boolean => {
|
||||
for (const element of subset) {
|
||||
if (!set.has(element)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
export const setIsEqual = <T>(setA: Set<T>, setB: Set<T>): boolean => {
|
||||
return setA.size === setB.size && setIsSuperset(setA, setB);
|
||||
};
|
||||
|
||||
export const handlePromiseError = <T>(promise: Promise<T>, logger: ImmichLogger): void => {
|
||||
promise.catch((error: Error | any) => logger.error(`Promise error: ${error}`, error?.stack));
|
||||
};
|
||||
@ -1,21 +0,0 @@
|
||||
import { FileValidator, Injectable } from '@nestjs/common';
|
||||
|
||||
@Injectable()
|
||||
export default class FileNotEmptyValidator extends FileValidator {
|
||||
constructor(private requiredFields: string[]) {
|
||||
super({});
|
||||
this.requiredFields = requiredFields;
|
||||
}
|
||||
|
||||
isValid(files?: any): boolean {
|
||||
if (!files) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return this.requiredFields.every((field) => files[field]);
|
||||
}
|
||||
|
||||
buildErrorMessage(): string {
|
||||
return `Field(s) ${this.requiredFields.join(', ')} should not be empty`;
|
||||
}
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
import { ArgumentMetadata, Injectable, ParseUUIDPipe } from '@nestjs/common';
|
||||
|
||||
@Injectable()
|
||||
export class ParseMeUUIDPipe extends ParseUUIDPipe {
|
||||
async transform(value: string, metadata: ArgumentMetadata) {
|
||||
if (value == 'me') {
|
||||
return value;
|
||||
}
|
||||
return super.transform(value, metadata);
|
||||
}
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { IsNotEmpty, IsUUID } from 'class-validator';
|
||||
|
||||
export class UUIDParamDto {
|
||||
@IsNotEmpty()
|
||||
@IsUUID('4')
|
||||
@ApiProperty({ format: 'uuid' })
|
||||
id!: string;
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
import { SetMetadata } from '@nestjs/common';
|
||||
|
||||
export const GENERATE_SQL_KEY = 'generate-sql-key';
|
||||
|
||||
export interface GenerateSqlQueries {
|
||||
name?: string;
|
||||
params: unknown[];
|
||||
}
|
||||
|
||||
/** Decorator to enable versioning/tracking of generated Sql */
|
||||
export const GenerateSql = (...options: GenerateSqlQueries[]) => SetMetadata(GENERATE_SQL_KEY, options);
|
||||
|
||||
const UUID = '00000000-0000-4000-a000-000000000000';
|
||||
|
||||
export const DummyValue = {
|
||||
UUID,
|
||||
UUID_SET: new Set([UUID]),
|
||||
PAGINATION: { take: 10, skip: 0 },
|
||||
EMAIL: 'user@immich.app',
|
||||
STRING: 'abcdefghi',
|
||||
BUFFER: Buffer.from('abcdefghi'),
|
||||
DATE: new Date(),
|
||||
TIME_BUCKET: '2024-01-01T00:00:00.000Z',
|
||||
};
|
||||
|
||||
// PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the
|
||||
// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching
|
||||
// by a list of IDs) requires splitting the query into multiple chunks.
|
||||
// We are rounding down this limit, as queries commonly include other filters and parameters.
|
||||
export const DATABASE_PARAMETER_CHUNK_SIZE = 65_500;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue