sascha
/
nextcloud-server
mirror of https://github.com/nextcloud/server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #54125 from nextcloud/s3-multipart-size-check

pull/54165/head
Kate 2025-08-05 20:57:15 +07:00 committed by GitHub
parent c695bf1a95 97efc95efc
commit 51e5f7b159
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 56 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/dav/lib/Connector/Sabre/File.php
Unescape Escape View File

@ -204,6 +204,9 @@ class File extends Node implements IFile {
}
}
$lengthHeader = $this->request->getHeader('content-length');
$expected = $lengthHeader !== '' ? (int)$lengthHeader : null;
if ($partStorage->instanceOfStorage(IWriteStreamStorage::class)) {
$isEOF = false;
$wrappedData = CallbackWrapper::wrap($data, null, null, null, null, function ($stream) use (&$isEOF): void {
@ -215,7 +218,7 @@ class File extends Node implements IFile {
$count = -1;
try {
/** @var IWriteStreamStorage $partStorage */
$count = $partStorage->writeStream($internalPartPath, $wrappedData);
$count = $partStorage->writeStream($internalPartPath, $wrappedData, $expected);
} catch (GenericFileException $e) {
$logger = Server::get(LoggerInterface::class);
$logger->error('Error while writing stream to storage: ' . $e->getMessage(), ['exception' => $e, 'app' => 'webdav']);
@ -235,10 +238,7 @@ class File extends Node implements IFile {
[$count, $result] = Files::streamCopy($data, $target, true);
fclose($target);
}
$lengthHeader = $this->request->getHeader('content-length');
$expected = $lengthHeader !== '' ? (int)$lengthHeader : -1;
if ($result === false && $expected >= 0) {
if ($result === false && $expected !== null) {
throw new Exception(
$this->l10n->t(
'Error while copying file to target location (copied: %1$s, expected filesize: %2$s)',
@ -253,7 +253,7 @@ class File extends Node implements IFile {
// if content length is sent by client:
// double check if the file was fully received
// compare expected and actual size
if ($expected >= 0
if ($expected !== null
&& $expected !== $count
&& $this->request->getMethod() === 'PUT'
) {

42
lib/private/Files/ObjectStore/ObjectStoreStorage.php
Unescape Escape View File

@ -475,6 +475,9 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
'original-storage' => $this->getId(),
'original-path' => $path,
];
if ($size) {
$metadata['size'] = $size;
}
$stat['mimetype'] = $mimetype;
$stat['etag'] = $this->getETag($path);
@ -496,32 +499,27 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
$urn = $this->getURN($fileId);
try {
//upload to object storage
if ($size === null) {
$countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) {
$totalWritten = 0;
$countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, $size, $exists, &$totalWritten) {
if (is_null($size) && !$exists) {
$this->getCache()->update($fileId, [
'size' => $writtenSize,
]);
$size = $writtenSize;
});
if ($this->objectStore instanceof IObjectStoreMetaData) {
$this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata);
} else {
$this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']);
}
if (is_resource($countStream)) {
fclose($countStream);
}
$stat['size'] = $size;
$totalWritten = $writtenSize;
});
if ($this->objectStore instanceof IObjectStoreMetaData) {
$this->objectStore->writeObjectWithMetaData($urn, $countStream, $metadata);
} else {
if ($this->objectStore instanceof IObjectStoreMetaData) {
$this->objectStore->writeObjectWithMetaData($urn, $stream, $metadata);
} else {
$this->objectStore->writeObject($urn, $stream, $metadata['mimetype']);
}
if (is_resource($stream)) {
fclose($stream);
}
$this->objectStore->writeObject($urn, $countStream, $metadata['mimetype']);
}
if (is_resource($countStream)) {
fclose($countStream);
}
$stat['size'] = $totalWritten;
} catch (\Exception $ex) {
if (!$exists) {
/*
@ -545,7 +543,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
]
);
}
throw $ex; // make this bubble up
throw new GenericFileException('Error while writing stream to object store', 0, $ex);
}
if ($exists) {
@ -561,7 +559,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFil
}
}
return $size;
return $totalWritten;
}
public function getObjectStore(): IObjectStore {

33
lib/private/Files/ObjectStore/S3ObjectTrait.php
Unescape Escape View File

@ -6,6 +6,8 @@
*/
namespace OC\Files\ObjectStore;
use Aws\Command;
use Aws\Exception\MultipartUploadException;
use Aws\S3\Exception\S3MultipartUploadException;
use Aws\S3\MultipartCopy;
use Aws\S3\MultipartUploader;
@ -96,7 +98,9 @@ trait S3ObjectTrait {
protected function writeSingle(string $urn, StreamInterface $stream, array $metaData): void {
$mimetype = $metaData['mimetype'] ?? null;
unset($metaData['mimetype']);
$this->getConnection()->putObject([
unset($metaData['size']);
$args = [
'Bucket' => $this->bucket,
'Key' => $urn,
'Body' => $stream,
@ -104,7 +108,13 @@ trait S3ObjectTrait {
'ContentType' => $mimetype,
'Metadata' => $this->buildS3Metadata($metaData),
'StorageClass' => $this->storageClass,
] + $this->getSSECParameters());
] + $this->getSSECParameters();
if ($size = $stream->getSize()) {
$args['ContentLength'] = $size;
}
$this->getConnection()->putObject($args);
}
@ -119,12 +129,15 @@ trait S3ObjectTrait {
protected function writeMultiPart(string $urn, StreamInterface $stream, array $metaData): void {
$mimetype = $metaData['mimetype'] ?? null;
unset($metaData['mimetype']);
unset($metaData['size']);
$attempts = 0;
$uploaded = false;
$concurrency = $this->concurrency;
$exception = null;
$state = null;
$size = $stream->getSize();
$totalWritten = 0;
// retry multipart upload once with concurrency at half on failure
while (!$uploaded && $attempts <= 1) {
@ -139,6 +152,15 @@ trait S3ObjectTrait {
'Metadata' => $this->buildS3Metadata($metaData),
'StorageClass' => $this->storageClass,
] + $this->getSSECParameters(),
'before_upload' => function (Command $command) use (&$totalWritten) {
$totalWritten += $command['ContentLength'];
},
'before_complete' => function ($_command) use (&$totalWritten, $size, &$uploader, &$attempts) {
if ($size !== null && $totalWritten != $size) {
$e = new \Exception('Incomplete multi part upload, expected ' . $size . ' bytes, wrote ' . $totalWritten);
throw new MultipartUploadException($uploader->getState(), $e);
}
},
]);
try {
@ -155,6 +177,9 @@ trait S3ObjectTrait {
if ($stream->isSeekable()) {
$stream->rewind();
}
} catch (MultipartUploadException $e) {
$exception = $e;
break;
}
}
@ -180,7 +205,9 @@ trait S3ObjectTrait {
public function writeObjectWithMetaData(string $urn, $stream, array $metaData): void {
$canSeek = fseek($stream, 0, SEEK_CUR) === 0;
$psrStream = Utils::streamFor($stream);
$psrStream = Utils::streamFor($stream, [
'size' => $metaData['size'] ?? null,
]);
$size = $psrStream->getSize();