Skip to content

Commit 0d9228f

Browse files
christyjacob4christyjacob4
authored
Merge pull request #121 from utopia-php/fix-resumable-upload-backport
S3 retry backported to
2 parents 893ccf0 + 833d429 commit 0d9228f

File tree

4 files changed

+142
-10
lines changed

4 files changed

+142
-10
lines changed

‎src/Storage/Device/Local.php‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,14 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks
9595
$tmp = \dirname($path).DIRECTORY_SEPARATOR.'tmp_'.\basename($path).DIRECTORY_SEPARATOR.\basename($path).'_chunks.log';
9696

9797
$this->createDirectory(\dirname($tmp));
98-
if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) {
99-
throw new Exception('Can\'t write chunk log '.$tmp);
98+
99+
$chunkFilePath = dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk;
100+
101+
// skip writing chunk if the chunk was re-uploaded
102+
if (! file_exists($chunkFilePath)) {
103+
if (! file_put_contents($tmp, "$chunk\n", FILE_APPEND)) {
104+
throw new Exception('Can\'t write chunk log '.$tmp);
105+
}
100106
}
101107

102108
$chunkLogs = file($tmp);
@@ -106,7 +112,7 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks
106112

107113
$chunksReceived = count(file($tmp));
108114

109-
if (! \rename($source, dirname($tmp).DIRECTORY_SEPARATOR.pathinfo($path, PATHINFO_FILENAME).'.part.'.$chunk)) {
115+
if (! \rename($source, $chunkFilePath)) {
110116
throw new Exception('Failed to write chunk '.$chunk);
111117
}
112118

‎src/Storage/Device/S3.php‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -315,11 +315,15 @@ public function uploadData(string $data, string $path, string $contentType, int
315315
$metadata['uploadId'] = $uploadId;
316316
}
317317

318-
$etag = $this->uploadPart($data, $path, $contentType, $chunk, $uploadId);
319318
$metadata['parts'] ??= [];
320-
$metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag];
321319
$metadata['chunks'] ??= 0;
322-
$metadata['chunks']++;
320+
321+
$etag = $this->uploadPart($data, $path, $contentType, $chunk, $uploadId);
322+
// skip incrementing if the chunk was re-uploaded
323+
if (! array_key_exists($chunk, $metadata['parts'])) {
324+
$metadata['chunks']++;
325+
}
326+
$metadata['parts'][$chunk] = $etag;
323327
if ($metadata['chunks'] == $chunks) {
324328
$this->completeMultipartUpload($path, $uploadId, $metadata['parts']);
325329
}
@@ -430,8 +434,8 @@ protected function completeMultipartUpload(string $path, string $uploadId, array
430434
$uri = $path !== '' ? '/'.\str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
431435

432436
$body = '<CompleteMultipartUpload>';
433-
foreach ($parts as $part) {
434-
$body .= "<Part><ETag>{$part['etag']}</ETag><PartNumber>{$part['partNumber']}</PartNumber></Part>";
437+
foreach ($parts as $key => $etag) {
438+
$body .= "<Part><ETag>{$etag}</ETag><PartNumber>{$key}</PartNumber></Part>";
435439
}
436440
$body .= '</CompleteMultipartUpload>';
437441

‎tests/Storage/Device/LocalTest.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,60 @@ public function testPartUpload()
173173
return $dest;
174174
}
175175

176+
public function testPartUploadRetry()
177+
{
178+
$source = __DIR__.'/../../resources/disk-a/large_file.mp4';
179+
$dest = $this->object->getPath('uploaded2.mp4');
180+
$totalSize = \filesize($source);
181+
// AWS S3 requires each part to be at least 5MB except for last part
182+
$chunkSize = 5 * 1024 * 1024;
183+
184+
$chunks = ceil($totalSize / $chunkSize);
185+
186+
$chunk = 1;
187+
$start = 0;
188+
$handle = @fopen($source, 'rb');
189+
$op = __DIR__.'/chunkx.part';
190+
while ($start < $totalSize) {
191+
$contents = fread($handle, $chunkSize);
192+
$op = __DIR__.'/chunkx.part';
193+
$cc = fopen($op, 'wb');
194+
fwrite($cc, $contents);
195+
fclose($cc);
196+
$this->object->upload($op, $dest, $chunk, $chunks);
197+
$start += strlen($contents);
198+
$chunk++;
199+
if ($chunk == 2) {
200+
break;
201+
}
202+
fseek($handle, $start);
203+
}
204+
@fclose($handle);
205+
206+
$chunk = 1;
207+
$start = 0;
208+
// retry from first to make sure duplicate chunk re-upload works without issue
209+
$handle = @fopen($source, 'rb');
210+
$op = __DIR__.'/chunkx.part';
211+
while ($start < $totalSize) {
212+
$contents = fread($handle, $chunkSize);
213+
$op = __DIR__.'/chunkx.part';
214+
$cc = fopen($op, 'wb');
215+
fwrite($cc, $contents);
216+
fclose($cc);
217+
$this->object->upload($op, $dest, $chunk, $chunks);
218+
$start += strlen($contents);
219+
$chunk++;
220+
fseek($handle, $start);
221+
}
222+
@fclose($handle);
223+
224+
$this->assertEquals(\filesize($source), $this->object->getFileSize($dest));
225+
$this->assertEquals(\md5_file($source), $this->object->getFileHash($dest));
226+
227+
return $dest;
228+
}
229+
176230
public function testAbort()
177231
{
178232
$source = __DIR__.'/../../resources/disk-a/large_file.mp4';

‎tests/Storage/S3Base.php‎

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -274,8 +274,76 @@ public function testPartUpload()
274274
$cc = fopen($op, 'wb');
275275
fwrite($cc, $contents);
276276
fclose($cc);
277-
$etag = $this->object->upload($op, $dest, $chunk, $chunks, $metadata);
278-
$parts[] = ['partNumber' => $chunk, 'etag' => $etag];
277+
$this->object->upload($op, $dest, $chunk, $chunks, $metadata);
278+
$start += strlen($contents);
279+
$chunk++;
280+
fseek($handle, $start);
281+
}
282+
@fclose($handle);
283+
unlink($op);
284+
285+
$this->assertEquals(\filesize($source), $this->object->getFileSize($dest));
286+
287+
// S3 doesnt provide a method to get a proper MD5-hash of a file created using multipart upload
288+
// https://stackoverflow.com/questions/8618218/amazon-s3-checksum
289+
// More info on how AWS calculates ETag for multipart upload here
290+
// https://savjee.be/2015/10/Verifying-Amazon-S3-multi-part-uploads-with-ETag-hash/
291+
// TODO
292+
// $this->assertEquals(\md5_file($source), $this->object->getFileHash($dest));
293+
// $this->object->delete($dest);
294+
return $dest;
295+
}
296+
297+
public function testPartUploadRetry()
298+
{
299+
$source = __DIR__.'/../resources/disk-a/large_file.mp4';
300+
$dest = $this->object->getPath('uploaded.mp4');
301+
$totalSize = \filesize($source);
302+
// AWS S3 requires each part to be at least 5MB except for last part
303+
$chunkSize = 5 * 1024 * 1024;
304+
305+
$chunks = ceil($totalSize / $chunkSize);
306+
307+
$chunk = 1;
308+
$start = 0;
309+
310+
$metadata = [
311+
'parts' => [],
312+
'chunks' => 0,
313+
'uploadId' => null,
314+
'content_type' => \mime_content_type($source),
315+
];
316+
$handle = @fopen($source, 'rb');
317+
$op = __DIR__.'/chunk.part';
318+
while ($start < $totalSize) {
319+
$contents = fread($handle, $chunkSize);
320+
$op = __DIR__.'/chunk.part';
321+
$cc = fopen($op, 'wb');
322+
fwrite($cc, $contents);
323+
fclose($cc);
324+
$this->object->upload($op, $dest, $chunk, $chunks, $metadata);
325+
$start += strlen($contents);
326+
$chunk++;
327+
if ($chunk == 2) {
328+
break;
329+
}
330+
fseek($handle, $start);
331+
}
332+
@fclose($handle);
333+
unlink($op);
334+
335+
$chunk = 1;
336+
$start = 0;
337+
// retry from first to make sure duplicate chunk re-upload works without issue
338+
$handle = @fopen($source, 'rb');
339+
$op = __DIR__.'/chunk.part';
340+
while ($start < $totalSize) {
341+
$contents = fread($handle, $chunkSize);
342+
$op = __DIR__.'/chunk.part';
343+
$cc = fopen($op, 'wb');
344+
fwrite($cc, $contents);
345+
fclose($cc);
346+
$this->object->upload($op, $dest, $chunk, $chunks, $metadata);
279347
$start += strlen($contents);
280348
$chunk++;
281349
fseek($handle, $start);

0 commit comments

Comments
 (0)