diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index 114ee85622b..5d10931b727 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -49,6 +49,9 @@ OCA\Files\Command\Object\Delete OCA\Files\Command\Object\Get OCA\Files\Command\Object\Put + OCA\Files\Command\Object\Info + OCA\Files\Command\Object\ListObject + OCA\Files\Command\Object\Orphans diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php index 5922560f521..de41b717fcc 100644 --- a/apps/files/composer/composer/autoload_classmap.php +++ b/apps/files/composer/composer/autoload_classmap.php @@ -34,7 +34,10 @@ return array( 'OCA\\Files\\Command\\Move' => $baseDir . '/../lib/Command/Move.php', 'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php', 'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php', + 'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php', + 'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php', + 'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php', 'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php', 'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php', diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php index bf489b037f7..f226a656e0d 100644 --- a/apps/files/composer/composer/autoload_static.php +++ b/apps/files/composer/composer/autoload_static.php @@ -49,7 +49,10 @@ class ComposerStaticInitFiles 'OCA\\Files\\Command\\Move' => __DIR__ . '/..' . '/../lib/Command/Move.php', 'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php', 'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php', + 'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php', + 'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php', + 'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php', 'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php', 'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php', diff --git a/apps/files/lib/Command/Object/Info.php b/apps/files/lib/Command/Object/Info.php new file mode 100644 index 00000000000..383ffa9d0ef --- /dev/null +++ b/apps/files/lib/Command/Object/Info.php @@ -0,0 +1,80 @@ +setName('files:object:info') + ->setDescription('Get the metadata of an object') + ->addArgument('object', InputArgument::REQUIRED, 'Object to get') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $object = $input->getArgument('object'); + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('Configured object store does currently not support retrieve metadata'); + return self::FAILURE; + } + + if (!$objectStore->objectExists($object)) { + $output->writeln("Object $object does not exist"); + return self::FAILURE; + } + + try { + $meta = $objectStore->getObjectMetaData($object); + } catch (\Exception $e) { + $msg = $e->getMessage(); + $output->writeln("Failed to read $object from object store: $msg"); + return self::FAILURE; + } + + if ($input->getOption('output') === 'plain' && isset($meta['size'])) { + $meta['size'] = Util::humanFileSize($meta['size']); + } + if (isset($meta['mtime'])) { + $meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM); + } + if (!isset($meta['mimetype'])) { + $handle = $objectStore->readObject($object); + $head = fread($handle, 8192); + fclose($handle); + $meta['mimetype'] = $this->mimeTypeDetector->detectString($head); + } + + $this->writeArrayInOutputFormat($input, $output, $meta); + + return self::SUCCESS; + } + +} diff --git a/apps/files/lib/Command/Object/ListObject.php b/apps/files/lib/Command/Object/ListObject.php new file mode 100644 index 00000000000..b377f7ae3c1 --- /dev/null +++ b/apps/files/lib/Command/Object/ListObject.php @@ -0,0 +1,50 @@ +setName('files:object:list') + ->setDescription('List all objects in the object store') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('Configured object store does currently not support listing objects'); + return self::FAILURE; + } + $objects = $objectStore->listObjects(); + $objects = $this->objectUtils->formatObjects($objects, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN); + $this->writeStreamingTableInOutputFormat($input, $output, $objects, self::CHUNK_SIZE); + + return self::SUCCESS; + } +} diff --git a/apps/files/lib/Command/Object/ObjectUtil.php b/apps/files/lib/Command/Object/ObjectUtil.php index c4ab59608fb..5f053c2c42f 100644 --- a/apps/files/lib/Command/Object/ObjectUtil.php +++ b/apps/files/lib/Command/Object/ObjectUtil.php @@ -12,6 +12,7 @@ use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; use OCP\IDBConnection; +use OCP\Util; use Symfony\Component\Console\Output\OutputInterface; class ObjectUtil { @@ -91,4 +92,24 @@ class ObjectUtil { return $fileId; } + + public function formatObjects(\Iterator $objects, bool $humanOutput): \Iterator { + foreach ($objects as $object) { + yield $this->formatObject($object, $humanOutput); + } + } + + public function formatObject(array $object, bool $humanOutput): array { + $row = array_merge([ + 'urn' => $object['urn'], + ], ($object['metadata'] ?? [])); + + if ($humanOutput && isset($row['size'])) { + $row['size'] = Util::humanFileSize($row['size']); + } + if (isset($row['mtime'])) { + $row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM); + } + return $row; + } } diff --git a/apps/files/lib/Command/Object/Orphans.php b/apps/files/lib/Command/Object/Orphans.php new file mode 100644 index 00000000000..14878fde925 --- /dev/null +++ b/apps/files/lib/Command/Object/Orphans.php @@ -0,0 +1,79 @@ +query) { + $this->query = $this->connection->getQueryBuilder(); + $this->query->select('fileid') + ->from('filecache') + ->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id'))); + } + return $this->query; + } + + protected function configure(): void { + parent::configure(); + $this + ->setName('files:object:orphans') + ->setDescription('List all objects in the object store that don\'t have a matching entry in the database') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('Configured object store does currently not support listing objects'); + return self::FAILURE; + } + $prefixLength = strlen('urn:oid:'); + + $objects = $objectStore->listObjects('urn:oid:'); + $orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) { + $fileId = (int)substr($object['urn'], $prefixLength); + return !$this->fileIdInDb($fileId); + }); + + $orphans = $this->objectUtils->formatObjects($orphans, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN); + $this->writeStreamingTableInOutputFormat($input, $output, $orphans, self::CHUNK_SIZE); + + return self::SUCCESS; + } + + private function fileIdInDb(int $fileId): bool { + $query = $this->getQuery(); + $query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT); + $result = $query->executeQuery(); + return $result->fetchOne() !== false; + } +} diff --git a/core/Command/Base.php b/core/Command/Base.php index b915ae2ae4a..c9b6337b64a 100644 --- a/core/Command/Base.php +++ b/core/Command/Base.php @@ -88,6 +88,58 @@ class Base extends Command implements CompletionAwareInterface { } } + protected function writeStreamingTableInOutputFormat(InputInterface $input, OutputInterface $output, \Iterator $items, int $tableGroupSize): void { + switch ($input->getOption('output')) { + case self::OUTPUT_FORMAT_JSON: + case self::OUTPUT_FORMAT_JSON_PRETTY: + $this->writeStreamingJsonArray($input, $output, $items); + break; + default: + foreach ($this->chunkIterator($items, $tableGroupSize) as $chunk) { + $this->writeTableInOutputFormat($input, $output, $chunk); + } + break; + } + } + + protected function writeStreamingJsonArray(InputInterface $input, OutputInterface $output, \Iterator $items): void { + $first = true; + $outputType = $input->getOption('output'); + + $output->writeln('['); + foreach ($items as $item) { + if (!$first) { + $output->writeln(','); + } + if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) { + $output->write(json_encode($item, JSON_PRETTY_PRINT)); + } else { + $output->write(json_encode($item)); + } + $first = false; + } + $output->writeln("\n]"); + } + + public function chunkIterator(\Iterator $iterator, int $count): \Iterator { + $chunk = []; + + for ($i = 0; $iterator->valid(); $i++) { + $chunk[] = $iterator->current(); + $iterator->next(); + if (count($chunk) == $count) { + // Got a full chunk, yield and start a new one + yield $chunk; + $chunk = []; + } + } + + if (count($chunk)) { + // Yield the last chunk even if incomplete + yield $chunk; + } + } + /** * @param mixed $item diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 6555ebcea6b..3522f137e22 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1627,6 +1627,7 @@ return array( 'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php', 'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php', + 'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php', 'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php', 'OC\\Files\\ObjectStore\\ObjectStoreScanner' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php', 'OC\\Files\\ObjectStore\\ObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 0f770eb1bdc..c06f25dd25d 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1676,6 +1676,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php', 'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php', + 'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php', 'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php', 'OC\\Files\\ObjectStore\\ObjectStoreScanner' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php', 'OC\\Files\\ObjectStore\\ObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php', diff --git a/lib/private/Files/ObjectStore/IObjectStoreMetaData.php b/lib/private/Files/ObjectStore/IObjectStoreMetaData.php new file mode 100644 index 00000000000..dc6b55076ac --- /dev/null +++ b/lib/private/Files/ObjectStore/IObjectStoreMetaData.php @@ -0,0 +1,36 @@ + + * + * @since 32.0.0 + */ + public function listObjects(string $prefix = ''): \Iterator; +} diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 41ab75caf45..71633b443dd 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -3,6 +3,7 @@ * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later */ + namespace OC\Files\ObjectStore; use Aws\Result; @@ -10,7 +11,7 @@ use Exception; use OCP\Files\ObjectStore\IObjectStore; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; -class S3 implements IObjectStore, IObjectStoreMultiPartUpload { +class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData { use S3ConnectionTrait; use S3ObjectTrait; @@ -61,7 +62,7 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload { 'Key' => $urn, 'UploadId' => $uploadId, 'MaxParts' => 1000, - 'PartNumberMarker' => $partNumberMarker + 'PartNumberMarker' => $partNumberMarker, ] + $this->getSSECParameters()); $parts = array_merge($parts, $result->get('Parts') ?? []); $isTruncated = $result->get('IsTruncated'); @@ -89,7 +90,41 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload { $this->getConnection()->abortMultipartUpload([ 'Bucket' => $this->bucket, 'Key' => $urn, - 'UploadId' => $uploadId + 'UploadId' => $uploadId, ]); } + + public function getObjectMetaData(string $urn): array { + $object = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ] + $this->getSSECParameters())->toArray(); + return [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ]; + } + + public function listObjects(string $prefix = ''): \Iterator { + $results = $this->getConnection()->getPaginator('ListObjectsV2', [ + 'Bucket' => $this->bucket, + 'Prefix' => $prefix, + ] + $this->getSSECParameters()); + + foreach ($results as $result) { + if (is_array($result['Contents'])) { + foreach ($result['Contents'] as $object) { + yield [ + 'urn' => basename($object['Key']), + 'metadata' => [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ], + ]; + } + } + } + } }