From 5117f9ab1efe837fb7da2be3d9099ed9e04ab73d Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 2 Sep 2025 09:28:30 +0200 Subject: [PATCH 1/5] fix(FileAccess): Make getByAncestorInStorage sharding ready Signed-off-by: Marcel Klehr --- lib/private/Files/Cache/FileAccess.php | 31 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php index c3f3614f3ca..3274af976b2 100644 --- a/lib/private/Files/Cache/FileAccess.php +++ b/lib/private/Files/Cache/FileAccess.php @@ -114,22 +114,20 @@ class FileAccess implements IFileAccess { $path = $root['path'] === '' ? '' : $root['path'] . '/'; - $qb->selectDistinct('*') + $qb->selectDistinct('f.*') ->from('filecache', 'f') ->where($qb->expr()->like('f.path', $qb->createNamedParameter($this->connection->escapeLikeParameter($path) . '%'))) ->andWhere($qb->expr()->eq('f.storage', $qb->createNamedParameter($storageId))) - ->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT))); + ->andWhere($qb->expr()->gt('f.fileid', $qb->createNamedParameter($fileIdCursor, IQueryBuilder::PARAM_INT))) + ->hintShardKey('storage', $storageId); - if (!$endToEndEncrypted) { + if (!$endToEndEncrypted && $this->connection->getShardDefinition('filecache') === null) { // End to end encrypted files are descendants of a folder with encrypted=1 - // Use a subquery to check the `encrypted` status of the parent folder - $subQuery = $this->getQuery()->select('p.encrypted') - ->from('filecache', 'p') - ->andWhere($qb->expr()->eq('p.fileid', 'f.parent')) - ->getSQL(); + // We can only do this inner join if the filecache table is not sharded + $qb->innerJoin('f', 'filecache', 'f2', $qb->expr()->eq('f2.fileid', 'f.parent')); $qb->andWhere( - $qb->expr()->eq($qb->createFunction(sprintf('(%s)', $subQuery)), $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) + $qb->expr()->eq('f2.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)) ); } @@ -152,6 +150,21 @@ class FileAccess implements IFileAccess { /** @var array */ $row = $files->fetch() ) { + if (!$endToEndEncrypted && $this->connection->getShardDefinition('filecache') !== null) { + // End to end encrypted files are descendants of a folder with encrypted=1 + // If the filecache table is sharded we need to check individually if the parent is encrypted + $parentQuery = $this->getQuery(); + $parentQuery->select('encrypted')->from('filecache'); + $parentQuery->whereFileId($row['parent']); + $parentQuery->hintShardKey('storage', $storageId); + $parentQuery->setMaxResults(1); + $result = $parentQuery->executeQuery(); + $encrypted = $result->fetchOne(); + $result->closeCursor(); + if ($encrypted === 1) { + continue; + } + } yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); } From bb82cff328da2e7aa720e1774a913c89d450d353 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 18 Sep 2025 11:43:08 +0200 Subject: [PATCH 2/5] fix(FileAccess): Chunk parent query Signed-off-by: Marcel Klehr --- lib/private/Files/Cache/FileAccess.php | 46 ++++++++++++++++++-------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php index 3274af976b2..8054f486e02 100644 --- a/lib/private/Files/Cache/FileAccess.php +++ b/lib/private/Files/Cache/FileAccess.php @@ -146,26 +146,44 @@ class FileAccess implements IFileAccess { $qb->orderBy('f.fileid', 'ASC'); $files = $qb->executeQuery(); - while ( - /** @var array */ - $row = $files->fetch() - ) { - if (!$endToEndEncrypted && $this->connection->getShardDefinition('filecache') !== null) { - // End to end encrypted files are descendants of a folder with encrypted=1 - // If the filecache table is sharded we need to check individually if the parent is encrypted + if (!$endToEndEncrypted && $this->connection->getShardDefinition('filecache') !== null) { + // End to end encrypted files are descendants of a folder with encrypted=1 + // If the filecache table is sharded we need to check with a separate query if the parent is encrypted + $rows = []; + $i = 0; + do { + while (($rows[] = $files->fetch()) && $i < 1000) { + $i++; + } + $parents = array_map(function ($row) { + return $row['parent']; + }, $rows); + $parentQuery = $this->getQuery(); - $parentQuery->select('encrypted')->from('filecache'); - $parentQuery->whereFileId($row['parent']); + $parentQuery->select('fileid', 'encrypted')->from('filecache'); + $parentQuery->where($parentQuery->expr()->in('fileid', $parentQuery->createNamedParameter($parents, IQueryBuilder::PARAM_INT_ARRAY))); $parentQuery->hintShardKey('storage', $storageId); - $parentQuery->setMaxResults(1); $result = $parentQuery->executeQuery(); - $encrypted = $result->fetchOne(); + $parentRows = $result->fetchAll(); $result->closeCursor(); - if ($encrypted === 1) { - continue; + + $parentEncryptedByFileId = array_column($parentRows, 'encrypted', 'fileid'); + foreach ($rows as $row) { + if ($parentEncryptedByFileId[$row['fileid']] === 1) { + continue; + } + yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); } + $rows = []; + $i = 1; + } while ($rows[] = $files->fetch()); + } else { + while ( + /** @var array */ + $row = $files->fetch() + ) { + yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); } - yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); } $files->closeCursor(); From 0a1754beaf803df361a0cbdee63e9564dcb4b7a4 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 18 Sep 2025 12:23:16 +0200 Subject: [PATCH 3/5] fix(FileAccess): Set filecache size column in tests Signed-off-by: Marcel Klehr --- tests/lib/Files/Cache/FileAccessTest.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/lib/Files/Cache/FileAccessTest.php b/tests/lib/Files/Cache/FileAccessTest.php index 63577ea874c..9a786f9430a 100644 --- a/tests/lib/Files/Cache/FileAccessTest.php +++ b/tests/lib/Files/Cache/FileAccessTest.php @@ -211,6 +211,7 @@ class FileAccessTest extends TestCase { 'name' => $queryBuilder->createNamedParameter('files'), 'mimetype' => 1, 'encrypted' => 0, + 'size' => 1, ]) ->executeStatement(); @@ -224,6 +225,7 @@ class FileAccessTest extends TestCase { 'name' => $queryBuilder->createNamedParameter('documents'), 'mimetype' => 2, 'encrypted' => 1, + 'size' => 1, ]) ->executeStatement(); @@ -237,6 +239,7 @@ class FileAccessTest extends TestCase { 'name' => $queryBuilder->createNamedParameter('photos'), 'mimetype' => 3, 'encrypted' => 1, + 'size' => 1, ]) ->executeStatement(); @@ -250,6 +253,7 @@ class FileAccessTest extends TestCase { 'name' => $queryBuilder->createNamedParameter('endtoendencrypted'), 'mimetype' => 4, 'encrypted' => 0, + 'size' => 1, ]) ->executeStatement(); @@ -263,6 +267,7 @@ class FileAccessTest extends TestCase { 'name' => $queryBuilder->createNamedParameter('serversideencrypted'), 'mimetype' => 4, 'encrypted' => 1, + 'size' => 1, ]) ->executeStatement(); @@ -276,6 +281,7 @@ class FileAccessTest extends TestCase { 'name' => $queryBuilder->createNamedParameter('storage2'), 'mimetype' => 5, 'encrypted' => 0, + 'size' => 1, ]) ->executeStatement(); @@ -289,6 +295,7 @@ class FileAccessTest extends TestCase { 'name' => $queryBuilder->createNamedParameter('file'), 'mimetype' => 6, 'encrypted' => 0, + 'size' => 1, ]) ->executeStatement(); } From 38f2bf6f98fac8521958d9a5b1d923bb729cd619 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 18 Sep 2025 12:54:39 +0200 Subject: [PATCH 4/5] fix(FileAccess): Try to fix type error Signed-off-by: Marcel Klehr --- lib/private/Files/Cache/FileAccess.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php index 8054f486e02..19f4c99c8b0 100644 --- a/lib/private/Files/Cache/FileAccess.php +++ b/lib/private/Files/Cache/FileAccess.php @@ -152,8 +152,9 @@ class FileAccess implements IFileAccess { $rows = []; $i = 0; do { - while (($rows[] = $files->fetch()) && $i < 1000) { + while ($i < 1000 && ($row = $files->fetch())) { $i++; + $rows[] = $row; } $parents = array_map(function ($row) { return $row['parent']; From 552ef445683f22661a5161208dac58ba7b066991 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 18 Sep 2025 13:36:22 +0200 Subject: [PATCH 5/5] fix(FileAccess): Make getAncestorInStorage pass sharding tests Signed-off-by: Marcel Klehr --- lib/private/Files/Cache/FileAccess.php | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/private/Files/Cache/FileAccess.php b/lib/private/Files/Cache/FileAccess.php index 19f4c99c8b0..86243b9ca25 100644 --- a/lib/private/Files/Cache/FileAccess.php +++ b/lib/private/Files/Cache/FileAccess.php @@ -150,10 +150,8 @@ class FileAccess implements IFileAccess { // End to end encrypted files are descendants of a folder with encrypted=1 // If the filecache table is sharded we need to check with a separate query if the parent is encrypted $rows = []; - $i = 0; do { - while ($i < 1000 && ($row = $files->fetch())) { - $i++; + while (count($rows) < 1000 && ($row = $files->fetch())) { $rows[] = $row; } $parents = array_map(function ($row) { @@ -168,15 +166,14 @@ class FileAccess implements IFileAccess { $parentRows = $result->fetchAll(); $result->closeCursor(); - $parentEncryptedByFileId = array_column($parentRows, 'encrypted', 'fileid'); + $encryptedByFileId = array_column($parentRows, 'encrypted', 'fileid'); foreach ($rows as $row) { - if ($parentEncryptedByFileId[$row['fileid']] === 1) { + if ($encryptedByFileId[$row['parent']]) { continue; } yield Cache::cacheEntryFromData($row, $this->mimeTypeLoader); } $rows = []; - $i = 1; } while ($rows[] = $files->fetch()); } else { while (