Merge pull request #52866 from nextcloud/backport/51603/stable31
[stable31] Add command to list orphan objectspull/50591/head
Robin Appelman
GitHub
commit
d18ff624d0
@ -0,0 +1,80 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
namespace OCA\Files\Command\Object;
|
||||
|
||||
use OC\Core\Command\Base;
|
||||
use OC\Files\ObjectStore\IObjectStoreMetaData;
|
||||
use OCP\Files\IMimeTypeDetector;
|
||||
use OCP\Util;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class Info extends Base {
|
||||
public function __construct(
|
||||
private ObjectUtil $objectUtils,
|
||||
private IMimeTypeDetector $mimeTypeDetector,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
parent::configure();
|
||||
$this
|
||||
->setName('files:object:info')
|
||||
->setDescription('Get the metadata of an object')
|
||||
->addArgument('object', InputArgument::REQUIRED, 'Object to get')
|
||||
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config");
|
||||
}
|
||||
|
||||
public function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$object = $input->getArgument('object');
|
||||
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
|
||||
if (!$objectStore) {
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if (!$objectStore instanceof IObjectStoreMetaData) {
|
||||
$output->writeln('<error>Configured object store does currently not support retrieve metadata</error>');
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if (!$objectStore->objectExists($object)) {
|
||||
$output->writeln("<error>Object $object does not exist</error>");
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
try {
|
||||
$meta = $objectStore->getObjectMetaData($object);
|
||||
} catch (\Exception $e) {
|
||||
$msg = $e->getMessage();
|
||||
$output->writeln("<error>Failed to read $object from object store: $msg</error>");
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if ($input->getOption('output') === 'plain' && isset($meta['size'])) {
|
||||
$meta['size'] = Util::humanFileSize($meta['size']);
|
||||
}
|
||||
if (isset($meta['mtime'])) {
|
||||
$meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM);
|
||||
}
|
||||
if (!isset($meta['mimetype'])) {
|
||||
$handle = $objectStore->readObject($object);
|
||||
$head = fread($handle, 8192);
|
||||
fclose($handle);
|
||||
$meta['mimetype'] = $this->mimeTypeDetector->detectString($head);
|
||||
}
|
||||
|
||||
$this->writeArrayInOutputFormat($input, $output, $meta);
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,50 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
namespace OCA\Files\Command\Object;
|
||||
|
||||
use OC\Core\Command\Base;
|
||||
use OC\Files\ObjectStore\IObjectStoreMetaData;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class ListObject extends Base {
|
||||
private const CHUNK_SIZE = 100;
|
||||
|
||||
public function __construct(
|
||||
private readonly ObjectUtil $objectUtils,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
parent::configure();
|
||||
$this
|
||||
->setName('files:object:list')
|
||||
->setDescription('List all objects in the object store')
|
||||
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
|
||||
}
|
||||
|
||||
public function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
|
||||
if (!$objectStore) {
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if (!$objectStore instanceof IObjectStoreMetaData) {
|
||||
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
|
||||
return self::FAILURE;
|
||||
}
|
||||
$objects = $objectStore->listObjects();
|
||||
$objects = $this->objectUtils->formatObjects($objects, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
|
||||
$this->writeStreamingTableInOutputFormat($input, $output, $objects, self::CHUNK_SIZE);
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,79 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
namespace OCA\Files\Command\Object;
|
||||
|
||||
use OC\Core\Command\Base;
|
||||
use OC\Files\ObjectStore\IObjectStoreMetaData;
|
||||
use OCP\DB\QueryBuilder\IQueryBuilder;
|
||||
use OCP\IDBConnection;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class Orphans extends Base {
|
||||
private const CHUNK_SIZE = 100;
|
||||
|
||||
private ?IQueryBuilder $query = null;
|
||||
|
||||
public function __construct(
|
||||
private readonly ObjectUtil $objectUtils,
|
||||
private readonly IDBConnection $connection,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
private function getQuery(): IQueryBuilder {
|
||||
if (!$this->query) {
|
||||
$this->query = $this->connection->getQueryBuilder();
|
||||
$this->query->select('fileid')
|
||||
->from('filecache')
|
||||
->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
|
||||
}
|
||||
return $this->query;
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
parent::configure();
|
||||
$this
|
||||
->setName('files:object:orphans')
|
||||
->setDescription('List all objects in the object store that don\'t have a matching entry in the database')
|
||||
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
|
||||
}
|
||||
|
||||
public function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
|
||||
if (!$objectStore) {
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if (!$objectStore instanceof IObjectStoreMetaData) {
|
||||
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
|
||||
return self::FAILURE;
|
||||
}
|
||||
$prefixLength = strlen('urn:oid:');
|
||||
|
||||
$objects = $objectStore->listObjects('urn:oid:');
|
||||
$orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) {
|
||||
$fileId = (int)substr($object['urn'], $prefixLength);
|
||||
return !$this->fileIdInDb($fileId);
|
||||
});
|
||||
|
||||
$orphans = $this->objectUtils->formatObjects($orphans, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
|
||||
$this->writeStreamingTableInOutputFormat($input, $output, $orphans, self::CHUNK_SIZE);
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
private function fileIdInDb(int $fileId): bool {
|
||||
$query = $this->getQuery();
|
||||
$query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
|
||||
$result = $query->executeQuery();
|
||||
return $result->fetchOne() !== false;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
namespace OC\Files\ObjectStore;
|
||||
|
||||
/**
|
||||
* Interface IObjectStoreMetaData
|
||||
*
|
||||
* @psalm-type ObjectMetaData = array{mtime?: \DateTime, etag?: string, size?: int, mimetype?: string, filename?: string}
|
||||
*/
|
||||
interface IObjectStoreMetaData {
|
||||
/**
|
||||
* Get metadata for an object.
|
||||
*
|
||||
* @param string $urn
|
||||
* @return ObjectMetaData
|
||||
*
|
||||
* @since 32.0.0
|
||||
*/
|
||||
public function getObjectMetaData(string $urn): array;
|
||||
|
||||
/**
|
||||
* List all objects in the object store.
|
||||
*
|
||||
* If the object store implementation can do it efficiently, the metadata for each object is also included.
|
||||
*
|
||||
* @param string $prefix
|
||||
* @return \Iterator<array{urn: string, metadata: ?ObjectMetaData}>
|
||||
*
|
||||
* @since 32.0.0
|
||||
*/
|
||||
public function listObjects(string $prefix = ''): \Iterator;
|
||||
}
|
||||
Loading…
Reference in New Issue