feat(server): conditionally run facial recognition nightly (#11080)

* only run nightly if new person

* add tests

* use string instead of date

* update sql

* update tests

* simplify condition

server/src/entities/system-metadata.entity.ts

@@ -12,6 +12,7 @@ export class SystemMetadataEntity<T extends keyof SystemMetadata = SystemMetadat
 
 export enum SystemMetadataKey {
   REVERSE_GEOCODING_STATE = 'reverse-geocoding-state',
+  FACIAL_RECOGNITION_STATE = 'facial-recognition-state',
   ADMIN_ONBOARDING = 'admin-onboarding',
   SYSTEM_CONFIG = 'system-config',
   VERSION_CHECK_STATE = 'version-check-state',
@@ -22,6 +23,7 @@ export type VersionCheckMetadata = { checkedAt: string; releaseVersion: string }
 
 export interface SystemMetadata extends Record<SystemMetadataKey, Record<string, any>> {
   [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
+  [SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string };
   [SystemMetadataKey.ADMIN_ONBOARDING]: { isOnboarded: boolean };
   [SystemMetadataKey.SYSTEM_CONFIG]: DeepPartial<SystemConfig>;
   [SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata;

server/src/interfaces/job.interface.ts

@@ -153,6 +153,10 @@ export interface IDeferrableJob extends IEntityJob {
   deferred?: boolean;
 }
 
+export interface INightlyJob extends IBaseJob {
+  nightly?: boolean;
+}
+
 export interface IEmailJob {
   to: string;
   subject: string;
@@ -229,7 +233,7 @@ export type JobItem =
   // Facial Recognition
   | { name: JobName.QUEUE_FACE_DETECTION; data: IBaseJob }
   | { name: JobName.FACE_DETECTION; data: IEntityJob }
-  | { name: JobName.QUEUE_FACIAL_RECOGNITION; data: IBaseJob }
+  | { name: JobName.QUEUE_FACIAL_RECOGNITION; data: INightlyJob }
   | { name: JobName.FACIAL_RECOGNITION; data: IDeferrableJob }
   | { name: JobName.GENERATE_PERSON_THUMBNAIL; data: IEntityJob }
 

server/src/interfaces/person.interface.ts

@@ -64,4 +64,5 @@ export interface IPersonRepository {
   getNumberOfPeople(userId: string): Promise<PeopleStatistics>;
   reassignFaces(data: UpdateFacesData): Promise<number>;
   update(entity: Partial<PersonEntity>): Promise<PersonEntity>;
+  getLatestFaceDate(): Promise<string | undefined>;
 }

server/src/queries/person.repository.sql

@@ -434,3 +434,9 @@ WHERE
   (("AssetFaceEntity"."personId" = $1))
 LIMIT
   1
+
+-- PersonRepository.getLatestFaceDate
+SELECT
+  MAX("jobStatus"."facesRecognizedAt")::text AS "latestDate"
+FROM
+  "asset_job_status" "jobStatus"

server/src/repositories/person.repository.ts

@@ -3,6 +3,7 @@ import { InjectRepository } from '@nestjs/typeorm';
 import _ from 'lodash';
 import { ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
+import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
 import { AssetEntity } from 'src/entities/asset.entity';
 import { PersonEntity } from 'src/entities/person.entity';
 import {
@@ -25,6 +26,7 @@ export class PersonRepository implements IPersonRepository {
     @InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
     @InjectRepository(PersonEntity) private personRepository: Repository<PersonEntity>,
     @InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
+    @InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository<AssetJobStatusEntity>,
   ) {}
 
   @GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] })
@@ -267,4 +269,13 @@ export class PersonRepository implements IPersonRepository {
   async getRandomFace(personId: string): Promise<AssetFaceEntity | null> {
     return this.assetFaceRepository.findOneBy({ personId });
   }
+
+  @GenerateSql()
+  async getLatestFaceDate(): Promise<string | undefined> {
+    const result: { latestDate?: string } | undefined = await this.jobStatusRepository
+      .createQueryBuilder('jobStatus')
+      .select('MAX(jobStatus.facesRecognizedAt)::text', 'latestDate')
+      .getRawOne();
+    return result?.latestDate;
+  }
 }

server/src/services/job.service.spec.ts

@@ -71,7 +71,7 @@ describe(JobService.name, () => {
         { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
         { name: JobName.CLEAN_OLD_AUDIT_LOGS },
         { name: JobName.USER_SYNC_USAGE },
-        { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } },
+        { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
         { name: JobName.CLEAN_OLD_SESSION_TOKENS },
       ]);
     });

server/src/services/job.service.ts

@@ -210,7 +210,7 @@ export class JobService {
       { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
       { name: JobName.CLEAN_OLD_AUDIT_LOGS },
       { name: JobName.USER_SYNC_USAGE },
-      { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } },
+      { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
       { name: JobName.CLEAN_OLD_SESSION_TOKENS },
     ]);
   }

server/src/services/person.service.spec.ts

@@ -3,6 +3,7 @@ import { Colorspace } from 'src/config';
 import { BulkIdErrorReason } from 'src/dtos/asset-ids.response.dto';
 import { PersonResponseDto, mapFaces, mapPerson } from 'src/dtos/person.dto';
 import { AssetFaceEntity } from 'src/entities/asset-face.entity';
+import { SystemMetadataKey } from 'src/entities/system-metadata.entity';
 import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
 import { ICryptoRepository } from 'src/interfaces/crypto.interface';
 import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface';
@@ -539,6 +540,7 @@ describe(PersonService.name, () => {
       await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(JobStatus.SKIPPED);
       expect(jobMock.queueAll).not.toHaveBeenCalled();
       expect(systemMock.get).toHaveBeenCalled();
+      expect(systemMock.set).not.toHaveBeenCalled();
     });
 
     it('should skip if recognition jobs are already queued', async () => {
@@ -546,6 +548,7 @@ describe(PersonService.name, () => {
 
       await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(JobStatus.SKIPPED);
       expect(jobMock.queueAll).not.toHaveBeenCalled();
+      expect(systemMock.set).not.toHaveBeenCalled();
     });
 
     it('should queue missing assets', async () => {
@@ -564,6 +567,9 @@ describe(PersonService.name, () => {
           data: { id: faceStub.face1.id, deferred: false },
         },
       ]);
+      expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE, {
+        lastRun: expect.any(String),
+      });
     });
 
     it('should queue all assets', async () => {
@@ -586,6 +592,59 @@ describe(PersonService.name, () => {
           data: { id: faceStub.face1.id, deferred: false },
         },
       ]);
+      expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE, {
+        lastRun: expect.any(String),
+      });
+    });
+
+    it('should run nightly if new face has been added since last run', async () => {
+      personMock.getLatestFaceDate.mockResolvedValue(new Date().toISOString());
+      personMock.getAllFaces.mockResolvedValue({
+        items: [faceStub.face1],
+        hasNextPage: false,
+      });
+      jobMock.getJobCounts.mockResolvedValue({ active: 1, waiting: 0, paused: 0, completed: 0, failed: 0, delayed: 0 });
+      personMock.getAll.mockResolvedValue({
+        items: [],
+        hasNextPage: false,
+      });
+      personMock.getAllFaces.mockResolvedValue({
+        items: [faceStub.face1],
+        hasNextPage: false,
+      });
+      await sut.handleQueueRecognizeFaces({ force: true, nightly: true });
+
+      expect(systemMock.get).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE);
+      expect(personMock.getLatestFaceDate).toHaveBeenCalledOnce();
+      expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.FACIAL_RECOGNITION,
+          data: { id: faceStub.face1.id, deferred: false },
+        },
+      ]);
+      expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE, {
+        lastRun: expect.any(String),
+      });
+    });
+
+    it('should skip nightly if no new face has been added since last run', async () => {
+      const lastRun = new Date();
+
+      systemMock.get.mockResolvedValue({ lastRun: lastRun.toISOString() });
+      personMock.getLatestFaceDate.mockResolvedValue(new Date(lastRun.getTime() - 1).toISOString());
+      personMock.getAllFaces.mockResolvedValue({
+        items: [faceStub.face1],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueRecognizeFaces({ force: true, nightly: true });
+
+      expect(systemMock.get).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE);
+      expect(personMock.getLatestFaceDate).toHaveBeenCalledOnce();
+      expect(personMock.getAllFaces).not.toHaveBeenCalled();
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
+      expect(systemMock.set).not.toHaveBeenCalled();
     });
 
     it('should delete existing people and faces if forced', async () => {

server/src/services/person.service.ts

@@ -26,6 +26,7 @@ import { AssetFaceEntity } from 'src/entities/asset-face.entity';
 import { AssetEntity, AssetType } from 'src/entities/asset.entity';
 import { PersonPathType } from 'src/entities/move.entity';
 import { PersonEntity } from 'src/entities/person.entity';
+import { SystemMetadataKey } from 'src/entities/system-metadata.entity';
 import { IAccessRepository } from 'src/interfaces/access.interface';
 import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
 import { ICryptoRepository } from 'src/interfaces/crypto.interface';
@@ -34,6 +35,7 @@ import {
   IDeferrableJob,
   IEntityJob,
   IJobRepository,
+  INightlyJob,
   JOBS_ASSET_PAGINATION_SIZE,
   JobItem,
   JobName,
@@ -67,7 +69,7 @@ export class PersonService {
     @Inject(IMoveRepository) moveRepository: IMoveRepository,
     @Inject(IMediaRepository) private mediaRepository: IMediaRepository,
     @Inject(IPersonRepository) private repository: IPersonRepository,
-    @Inject(ISystemMetadataRepository) systemMetadataRepository: ISystemMetadataRepository,
+    @Inject(ISystemMetadataRepository) private systemMetadataRepository: ISystemMetadataRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(ISearchRepository) private smartInfoRepository: ISearchRepository,
@@ -376,13 +378,26 @@ export class PersonService {
     return JobStatus.SUCCESS;
   }
 
-  async handleQueueRecognizeFaces({ force }: IBaseJob): Promise<JobStatus> {
+  async handleQueueRecognizeFaces({ force, nightly }: INightlyJob): Promise<JobStatus> {
     const { machineLearning } = await this.configCore.getConfig({ withCache: false });
     if (!isFacialRecognitionEnabled(machineLearning)) {
       return JobStatus.SKIPPED;
     }
 
     await this.jobRepository.waitForQueueCompletion(QueueName.THUMBNAIL_GENERATION, QueueName.FACE_DETECTION);
+
+    if (nightly) {
+      const [state, latestFaceDate] = await Promise.all([
+        this.systemMetadataRepository.get(SystemMetadataKey.FACIAL_RECOGNITION_STATE),
+        this.repository.getLatestFaceDate(),
+      ]);
+
+      if (state?.lastRun && latestFaceDate && state.lastRun > latestFaceDate) {
+        this.logger.debug('Skipping facial recognition nightly since no face has been added since the last run');
+        return JobStatus.SKIPPED;
+      }
+    }
+
     const { waiting } = await this.jobRepository.getJobCounts(QueueName.FACIAL_RECOGNITION);
 
     if (force) {
@@ -394,6 +409,7 @@ export class PersonService {
       return JobStatus.SKIPPED;
     }
 
+    const lastRun = new Date().toISOString();
     const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
       this.repository.getAllFaces(pagination, { where: force ? undefined : { personId: IsNull() } }),
     );
@@ -404,6 +420,8 @@ export class PersonService {
       );
     }
 
+    await this.systemMetadataRepository.set(SystemMetadataKey.FACIAL_RECOGNITION_STATE, { lastRun });
+
     return JobStatus.SUCCESS;
   }
 

server/test/repositories/person.repository.mock.ts

@@ -29,5 +29,6 @@ export const newPersonRepositoryMock = (): Mocked<IPersonRepository> => {
     getFaceById: vitest.fn(),
     getFaceByIdWithAssets: vitest.fn(),
     getNumberOfPeople: vitest.fn(),
+    getLatestFaceDate: vitest.fn(),
   };
 };