Merge pull request #52786 from nextcloud/multi-object-store
allow configuring multiple object store backendspull/54058/head
Robin Appelman
GitHub
commit
b69109395b
@ -0,0 +1,108 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl>
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
namespace OCA\Files\Command\Object\Multi;
|
||||
|
||||
use OC\Core\Command\Base;
|
||||
use OC\Files\ObjectStore\PrimaryObjectStoreConfig;
|
||||
use OCP\IConfig;
|
||||
use OCP\IDBConnection;
|
||||
use Symfony\Component\Console\Helper\QuestionHelper;
|
||||
use Symfony\Component\Console\Input\InputArgument;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Question\ConfirmationQuestion;
|
||||
|
||||
class Rename extends Base {
|
||||
public function __construct(
|
||||
private readonly IDBConnection $connection,
|
||||
private readonly PrimaryObjectStoreConfig $objectStoreConfig,
|
||||
private readonly IConfig $config,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
parent::configure();
|
||||
$this
|
||||
->setName('files:object:multi:rename-config')
|
||||
->setDescription('Rename an object store configuration and move all users over to the new configuration,')
|
||||
->addArgument('source', InputArgument::REQUIRED, 'Object store configuration to rename')
|
||||
->addArgument('target', InputArgument::REQUIRED, 'New name for the object store configuration');
|
||||
}
|
||||
|
||||
public function execute(InputInterface $input, OutputInterface $output): int {
|
||||
$source = $input->getArgument('source');
|
||||
$target = $input->getArgument('target');
|
||||
|
||||
$configs = $this->objectStoreConfig->getObjectStoreConfigs();
|
||||
if (!isset($configs[$source])) {
|
||||
$output->writeln('<error>Unknown object store configuration: ' . $source . '</error>');
|
||||
return 1;
|
||||
}
|
||||
|
||||
if ($source === 'root') {
|
||||
$output->writeln('<error>Renaming the root configuration is not supported.</error>');
|
||||
return 1;
|
||||
}
|
||||
|
||||
if ($source === 'default') {
|
||||
$output->writeln('<error>Renaming the default configuration is not supported.</error>');
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (!isset($configs[$target])) {
|
||||
$output->writeln('<comment>Target object store configuration ' . $target . ' doesn\'t exist yet.</comment>');
|
||||
$output->writeln('The target configuration can be created automatically.');
|
||||
$output->writeln('However, as this depends on modifying the config.php, this only works as long as the instance runs on a single node or all nodes in a clustered setup have a shared config file (such as from a shared network mount).');
|
||||
$output->writeln('If the different nodes have a separate copy of the config.php file, the automatic object store configuration creation will lead to the configuration going out of sync.');
|
||||
$output->writeln('If these requirements are not met, you can manually create the target object store configuration in each node\'s configuration before running the command.');
|
||||
$output->writeln('');
|
||||
$output->writeln('<error>Failure to check these requirements will lead to data loss for users.</error>');
|
||||
|
||||
/** @var QuestionHelper $helper */
|
||||
$helper = $this->getHelper('question');
|
||||
$question = new ConfirmationQuestion('Automatically create target object store configuration? [y/N] ', false);
|
||||
if ($helper->ask($input, $output, $question)) {
|
||||
$configs[$target] = $configs[$source];
|
||||
|
||||
// update all aliases
|
||||
foreach ($configs as &$config) {
|
||||
if ($config === $source) {
|
||||
$config = $target;
|
||||
}
|
||||
}
|
||||
$this->config->setSystemValue('objectstore', $configs);
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
} elseif (($configs[$source] !== $configs[$target]) || $configs[$source] !== $target) {
|
||||
$output->writeln('<error>Source and target configuration differ.</error>');
|
||||
$output->writeln('');
|
||||
$output->writeln('To ensure proper migration of users, the source and target configuration must be the same to ensure that the objects for the moved users exist on the target configuration.');
|
||||
$output->writeln('The usual migration process consists of creating a clone of the old configuration, moving the users from the old configuration to the new one, and then adjust the old configuration that is longer used.');
|
||||
return 1;
|
||||
}
|
||||
|
||||
$query = $this->connection->getQueryBuilder();
|
||||
$query->update('preferences')
|
||||
->set('configvalue', $query->createNamedParameter($target))
|
||||
->where($query->expr()->eq('appid', $query->createNamedParameter('homeobjectstore')))
|
||||
->andWhere($query->expr()->eq('configkey', $query->createNamedParameter('objectstore')))
|
||||
->andWhere($query->expr()->eq('configvalue', $query->createNamedParameter($source)));
|
||||
$count = $query->executeStatement();
|
||||
|
||||
if ($count > 0) {
|
||||
$output->writeln('Moved <info>' . $count . '</info> users');
|
||||
} else {
|
||||
$output->writeln('No users moved');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,98 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl>
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
namespace OCA\Files\Command\Object\Multi;
|
||||
|
||||
use OC\Core\Command\Base;
|
||||
use OC\Files\ObjectStore\PrimaryObjectStoreConfig;
|
||||
use OCP\IConfig;
|
||||
use OCP\IUser;
|
||||
use OCP\IUserManager;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class Users extends Base {
|
||||
public function __construct(
|
||||
private readonly IUserManager $userManager,
|
||||
private readonly PrimaryObjectStoreConfig $objectStoreConfig,
|
||||
private readonly IConfig $config,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void {
|
||||
parent::configure();
|
||||
$this
|
||||
->setName('files:object:multi:users')
|
||||
->setDescription('Get the mapping between users and object store buckets')
|
||||
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, 'Only list users using the specified bucket')
|
||||
->addOption('object-store', 'o', InputOption::VALUE_REQUIRED, 'Only list users using the specified object store configuration')
|
||||
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Only show the mapping for the specified user, ignores all other options');
|
||||
}
|
||||
|
||||
public function execute(InputInterface $input, OutputInterface $output): int {
|
||||
if ($userId = $input->getOption('user')) {
|
||||
$user = $this->userManager->get($userId);
|
||||
if (!$user) {
|
||||
$output->writeln("<error>User $userId not found</error>");
|
||||
return 1;
|
||||
}
|
||||
$users = new \ArrayIterator([$user]);
|
||||
} else {
|
||||
$bucket = (string)$input->getOption('bucket');
|
||||
$objectStore = (string)$input->getOption('object-store');
|
||||
if ($bucket !== '' && $objectStore === '') {
|
||||
$users = $this->getUsers($this->config->getUsersForUserValue('homeobjectstore', 'bucket', $bucket));
|
||||
} elseif ($bucket === '' && $objectStore !== '') {
|
||||
$users = $this->getUsers($this->config->getUsersForUserValue('homeobjectstore', 'objectstore', $objectStore));
|
||||
} elseif ($bucket) {
|
||||
$users = $this->getUsers(array_intersect(
|
||||
$this->config->getUsersForUserValue('homeobjectstore', 'bucket', $bucket),
|
||||
$this->config->getUsersForUserValue('homeobjectstore', 'objectstore', $objectStore)
|
||||
));
|
||||
} else {
|
||||
$users = $this->userManager->getSeenUsers();
|
||||
}
|
||||
}
|
||||
|
||||
$this->writeStreamingTableInOutputFormat($input, $output, $this->infoForUsers($users), 100);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string[] $userIds
|
||||
* @return \Iterator<IUser>
|
||||
*/
|
||||
private function getUsers(array $userIds): \Iterator {
|
||||
foreach ($userIds as $userId) {
|
||||
$user = $this->userManager->get($userId);
|
||||
if ($user) {
|
||||
yield $user;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param \Iterator<IUser> $users
|
||||
* @return \Iterator<array>
|
||||
*/
|
||||
private function infoForUsers(\Iterator $users): \Iterator {
|
||||
foreach ($users as $user) {
|
||||
yield $this->infoForUser($user);
|
||||
}
|
||||
}
|
||||
|
||||
private function infoForUser(IUser $user): array {
|
||||
return [
|
||||
'user' => $user->getUID(),
|
||||
'object-store' => $this->objectStoreConfig->getObjectStoreForUser($user),
|
||||
'bucket' => $this->objectStoreConfig->getSetBucketForUser($user) ?? 'unset',
|
||||
];
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl>
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
namespace OC\Files\ObjectStore;
|
||||
|
||||
class InvalidObjectStoreConfigurationException extends \Exception {
|
||||
|
||||
}
|
||||
@ -0,0 +1,285 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* SPDX-FileCopyrightText: 2025 Robin Appelman <robin@icewind.nl>
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
namespace lib\Files\ObjectStore;
|
||||
|
||||
use OC\Files\ObjectStore\PrimaryObjectStoreConfig;
|
||||
use OC\Files\ObjectStore\StorageObjectStore;
|
||||
use OCP\App\IAppManager;
|
||||
use OCP\IConfig;
|
||||
use OCP\IUser;
|
||||
use PHPUnit\Framework\MockObject\MockObject;
|
||||
use Test\TestCase;
|
||||
|
||||
class PrimaryObjectStoreConfigTest extends TestCase {
|
||||
private array $systemConfig = [];
|
||||
private array $userConfig = [];
|
||||
private IConfig&MockObject $config;
|
||||
private IAppManager&MockObject $appManager;
|
||||
private PrimaryObjectStoreConfig $objectStoreConfig;
|
||||
|
||||
protected function setUp(): void {
|
||||
parent::setUp();
|
||||
|
||||
$this->systemConfig = [];
|
||||
$this->config = $this->createMock(IConfig::class);
|
||||
$this->appManager = $this->createMock(IAppManager::class);
|
||||
$this->config->method('getSystemValue')
|
||||
->willReturnCallback(function ($key, $default = '') {
|
||||
if (isset($this->systemConfig[$key])) {
|
||||
return $this->systemConfig[$key];
|
||||
} else {
|
||||
return $default;
|
||||
}
|
||||
});
|
||||
$this->config->method('getUserValue')
|
||||
->willReturnCallback(function ($userId, $appName, $key, $default = '') {
|
||||
if (isset($this->userConfig[$userId][$appName][$key])) {
|
||||
return $this->userConfig[$userId][$appName][$key];
|
||||
} else {
|
||||
return $default;
|
||||
}
|
||||
});
|
||||
$this->config->method('setUserValue')
|
||||
->willReturnCallback(function ($userId, $appName, $key, $value) {
|
||||
$this->userConfig[$userId][$appName][$key] = $value;
|
||||
});
|
||||
|
||||
$this->objectStoreConfig = new PrimaryObjectStoreConfig($this->config, $this->appManager);
|
||||
}
|
||||
|
||||
private function getUser(string $uid): IUser {
|
||||
$user = $this->createMock(IUser::class);
|
||||
$user->method('getUID')
|
||||
->willReturn($uid);
|
||||
return $user;
|
||||
}
|
||||
|
||||
private function setConfig(string $key, $value) {
|
||||
$this->systemConfig[$key] = $value;
|
||||
}
|
||||
|
||||
public function testNewUserGetsDefault() {
|
||||
$this->setConfig('objectstore', [
|
||||
'default' => 'server1',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test'));
|
||||
$this->assertEquals('server1', $result['arguments']['host']);
|
||||
|
||||
$this->assertEquals('server1', $this->config->getUserValue('test', 'homeobjectstore', 'objectstore', null));
|
||||
}
|
||||
|
||||
public function testExistingUserKeepsStorage() {
|
||||
// setup user with `server1` as storage
|
||||
$this->testNewUserGetsDefault();
|
||||
|
||||
$this->setConfig('objectstore', [
|
||||
'default' => 'server2',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
],
|
||||
],
|
||||
'server2' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server2',
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test'));
|
||||
$this->assertEquals('server1', $result['arguments']['host']);
|
||||
|
||||
$this->assertEquals('server1', $this->config->getUserValue('test', 'homeobjectstore', 'objectstore', null));
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('other-user'));
|
||||
$this->assertEquals('server2', $result['arguments']['host']);
|
||||
}
|
||||
|
||||
public function testNestedAliases() {
|
||||
$this->setConfig('objectstore', [
|
||||
'default' => 'a1',
|
||||
'a1' => 'a2',
|
||||
'a2' => 'server1',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
],
|
||||
],
|
||||
]);
|
||||
$this->assertEquals('server1', $this->objectStoreConfig->resolveAlias('default'));
|
||||
}
|
||||
|
||||
public function testMultibucketChangedConfig() {
|
||||
$this->setConfig('objectstore', [
|
||||
'default' => 'server1',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
'multibucket' => true,
|
||||
'num_buckets' => 8,
|
||||
'bucket' => 'bucket-'
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test'));
|
||||
$this->assertEquals('server1', $result['arguments']['host']);
|
||||
$this->assertEquals('bucket-7', $result['arguments']['bucket']);
|
||||
|
||||
$this->setConfig('objectstore', [
|
||||
'default' => 'server1',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
'multibucket' => true,
|
||||
'num_buckets' => 64,
|
||||
'bucket' => 'bucket-'
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test'));
|
||||
$this->assertEquals('server1', $result['arguments']['host']);
|
||||
$this->assertEquals('bucket-7', $result['arguments']['bucket']);
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test-foo'));
|
||||
$this->assertEquals('server1', $result['arguments']['host']);
|
||||
$this->assertEquals('bucket-40', $result['arguments']['bucket']);
|
||||
|
||||
$this->setConfig('objectstore', [
|
||||
'default' => 'server2',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
'multibucket' => true,
|
||||
'num_buckets' => 64,
|
||||
'bucket' => 'bucket-'
|
||||
],
|
||||
],
|
||||
'server2' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server2',
|
||||
'multibucket' => true,
|
||||
'num_buckets' => 16,
|
||||
'bucket' => 'bucket-'
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test'));
|
||||
$this->assertEquals('server1', $result['arguments']['host']);
|
||||
$this->assertEquals('bucket-7', $result['arguments']['bucket']);
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForUser($this->getUser('test-bar'));
|
||||
$this->assertEquals('server2', $result['arguments']['host']);
|
||||
$this->assertEquals('bucket-4', $result['arguments']['bucket']);
|
||||
}
|
||||
|
||||
public function testMultibucketOldConfig() {
|
||||
$this->setConfig('objectstore_multibucket', [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
'multibucket' => true,
|
||||
'num_buckets' => 8,
|
||||
'bucket' => 'bucket-'
|
||||
],
|
||||
]);
|
||||
$configs = $this->objectStoreConfig->getObjectStoreConfigs();
|
||||
$this->assertEquals([
|
||||
'default' => 'server1',
|
||||
'root' => 'server1',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
'multibucket' => true,
|
||||
'num_buckets' => 8,
|
||||
'bucket' => 'bucket-'
|
||||
],
|
||||
],
|
||||
], $configs);
|
||||
}
|
||||
|
||||
public function testSingleObjectStore() {
|
||||
$this->setConfig('objectstore', [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
],
|
||||
]);
|
||||
$configs = $this->objectStoreConfig->getObjectStoreConfigs();
|
||||
$this->assertEquals([
|
||||
'default' => 'server1',
|
||||
'root' => 'server1',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
'multibucket' => false,
|
||||
],
|
||||
],
|
||||
], $configs);
|
||||
}
|
||||
|
||||
public function testRoot() {
|
||||
$this->setConfig('objectstore', [
|
||||
'default' => 'server1',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
],
|
||||
],
|
||||
'server2' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server2',
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForRoot();
|
||||
$this->assertEquals('server1', $result['arguments']['host']);
|
||||
|
||||
$this->setConfig('objectstore', [
|
||||
'default' => 'server1',
|
||||
'root' => 'server2',
|
||||
'server1' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server1',
|
||||
],
|
||||
],
|
||||
'server2' => [
|
||||
'class' => StorageObjectStore::class,
|
||||
'arguments' => [
|
||||
'host' => 'server2',
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$result = $this->objectStoreConfig->getObjectStoreConfigForRoot();
|
||||
$this->assertEquals('server2', $result['arguments']['host']);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue