diff --git a/server/src/queries/person.repository.sql b/server/src/queries/person.repository.sql
index 8ad5b96bbc..d987bf3f3f 100644
--- a/server/src/queries/person.repository.sql
+++ b/server/src/queries/person.repository.sql
@@ -259,8 +259,13 @@ where
           and "asset"."visibility" = 'timeline'
           and "asset"."deletedAt" is null
       )
+    having
+      (
+        "person"."name" != $2
+        or count("asset_face"."assetId") >= $3
+      )
   )
-  and "person"."ownerId" = $2
+  and "person"."ownerId" = $4
 
 -- PersonRepository.refreshFaces
 with