From 16da8bbc8afc06b7807c8b9ce05b4ae8e678ee1d Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 14 Oct 2025 09:30:42 +0200 Subject: [PATCH] fix(TaskProcessingApiController): Implement getNextScheduledTasks for next_batch endpoint in order to avoid hitting the DB with multiple 1 row requests Signed-off-by: Marcel Klehr --- .../TaskProcessingApiController.php | 26 +++++----------- lib/private/TaskProcessing/Db/TaskMapper.php | 31 +++++++++++++++++++ lib/private/TaskProcessing/Manager.php | 12 +++++++ lib/public/TaskProcessing/IManager.php | 10 ++++++ 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/core/Controller/TaskProcessingApiController.php b/core/Controller/TaskProcessingApiController.php index a241427cadc..ed0351f80d4 100644 --- a/core/Controller/TaskProcessingApiController.php +++ b/core/Controller/TaskProcessingApiController.php @@ -576,6 +576,7 @@ class TaskProcessingApiController extends OCSController { /** * Returns the next n scheduled tasks for the specified set of taskTypes and providers + * The returned tasks are capped at ~50MiB * * @param list $providerIds The ids of the providers * @param list $taskTypeIds The ids of the task types @@ -597,23 +598,18 @@ class TaskProcessingApiController extends OCSController { ]); } - /** @var list $tasks */ - $tasks = []; - $taskIdsToIgnore = []; + $tasks = $this->taskProcessingManager->getNextScheduledTasks($possibleTaskTypeIds, numberOfTasks: $numberOfTasks + 1); + $tasksJson = []; // Stop when $numberOfTasks is reached or the json payload is larger than 50MiB - while (count($tasks) < $numberOfTasks && strlen(json_encode($tasks)) < 50 * 1024 * 1024) { + while (count($tasks) > 0 && count($tasksJson) < $numberOfTasks && strlen(json_encode($tasks)) < 50 * 1024 * 1024) { // Until we find a task whose task type is set to be provided by the providers requested with this request // Or no scheduled task is found anymore (given the taskIds to ignore) - try { - $task = $this->taskProcessingManager->getNextScheduledTask($possibleTaskTypeIds, $taskIdsToIgnore); - } catch (NotFoundException) { - break; - } + $task = array_shift($tasks); try { $provider = $this->taskProcessingManager->getPreferredProvider($task->getTaskTypeId()); if (in_array($provider->getId(), $possibleProviderIds, true)) { if ($this->taskProcessingManager->lockTask($task)) { - $tasks[] = ['task' => $task->jsonSerialize(), 'provider' => $provider->getId()]; + $tasksJson[] = ['task' => $task->jsonSerialize(), 'provider' => $provider->getId()]; continue; } } @@ -621,16 +617,8 @@ class TaskProcessingApiController extends OCSController { // There is no provider set for the task type of this task // proceed to ignore this task } - - $taskIdsToIgnore[] = (int)$task->getId(); - } - - try { - $this->taskProcessingManager->getNextScheduledTask($possibleTaskTypeIds, $taskIdsToIgnore); - $hasMore = true; - } catch (\Throwable) { - $hasMore = false; } + $hasMore = count($tasks) > 0; return new DataResponse([ 'tasks' => $tasks, diff --git a/lib/private/TaskProcessing/Db/TaskMapper.php b/lib/private/TaskProcessing/Db/TaskMapper.php index fee96534633..e235ff4ec0c 100644 --- a/lib/private/TaskProcessing/Db/TaskMapper.php +++ b/lib/private/TaskProcessing/Db/TaskMapper.php @@ -233,4 +233,35 @@ class TaskMapper extends QBMapper { return 0; } } + + /** + * @param list $taskTypes + * @param list $taskIdsToIgnore + * @param int $numberOfTasks + * @return list + * @throws Exception + */ + public function findNOldestScheduledByType(array $taskTypes, array $taskIdsToIgnore, int $numberOfTasks) { + $qb = $this->db->getQueryBuilder(); + $qb->select(Task::$columns) + ->from($this->tableName) + ->where($qb->expr()->eq('status', $qb->createPositionalParameter(\OCP\TaskProcessing\Task::STATUS_SCHEDULED, IQueryBuilder::PARAM_INT))) + ->setMaxResults($numberOfTasks) + ->orderBy('last_updated', 'ASC'); + + if (!empty($taskTypes)) { + $filter = []; + foreach ($taskTypes as $taskType) { + $filter[] = $qb->expr()->eq('type', $qb->createPositionalParameter($taskType)); + } + + $qb->andWhere($qb->expr()->orX(...$filter)); + } + + if (!empty($taskIdsToIgnore)) { + $qb->andWhere($qb->expr()->notIn('id', $qb->createNamedParameter($taskIdsToIgnore, IQueryBuilder::PARAM_INT_ARRAY))); + } + + return $this->findEntities($qb); + } } diff --git a/lib/private/TaskProcessing/Manager.php b/lib/private/TaskProcessing/Manager.php index 80b33e9b79b..e3f466755d6 100644 --- a/lib/private/TaskProcessing/Manager.php +++ b/lib/private/TaskProcessing/Manager.php @@ -1192,6 +1192,18 @@ class Manager implements IManager { } } + public function getNextScheduledTasks(array $taskTypeIds = [], array $taskIdsToIgnore = [], int $numberOfTasks = 1): array { + try { + return array_map(fn ($taskEntity) => $taskEntity->toPublicTask(), $this->taskMapper->findNOldestScheduledByType($taskTypeIds, $taskIdsToIgnore, $numberOfTasks); + } catch (DoesNotExistException $e) { + throw new \OCP\TaskProcessing\Exception\NotFoundException('Could not find the task', 0, $e); + } catch (\OCP\DB\Exception $e) { + throw new \OCP\TaskProcessing\Exception\Exception('There was a problem finding the task', 0, $e); + } catch (\JsonException $e) { + throw new \OCP\TaskProcessing\Exception\Exception('There was a problem parsing JSON after finding the task', 0, $e); + } + } + /** * Takes task input data and replaces fileIds with File objects * diff --git a/lib/public/TaskProcessing/IManager.php b/lib/public/TaskProcessing/IManager.php index c8f86364c35..28c99d5299b 100644 --- a/lib/public/TaskProcessing/IManager.php +++ b/lib/public/TaskProcessing/IManager.php @@ -160,6 +160,16 @@ interface IManager { */ public function getNextScheduledTask(array $taskTypeIds = [], array $taskIdsToIgnore = []): Task; + /** + * @param list $taskTypeIds + * @param list $taskIdsToIgnore + * @param int $numberOfTasks + * @return list + * @throws Exception If the query failed + * @since 33.0.0 + */ + public function getNextScheduledTasks(array $taskTypeIds = [], array $taskIdsToIgnore = [], int $numberOfTasks = 1): array; + /** * @param int $id The id of the task * @param string|null $userId The user id that scheduled the task