From b2d70f65b3ec04205eb09e09599a06da04186f83 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 17 Apr 2023 11:25:36 -0400
Subject: [PATCH 01/31] multipart uploader method
---
src/Multipart/AbstractUploader.php | 7 +++++++
src/Multipart/UploadState.php | 10 ++++++++-
src/S3/MultipartUploader.php | 33 ++++++++++++++++++++++++++++++
src/S3/MultipartUploadingTrait.php | 7 +++++++
src/S3/ObjectUploader.php | 18 ++++++++++++++--
5 files changed, 72 insertions(+), 3 deletions(-)
diff --git a/src/Multipart/AbstractUploader.php b/src/Multipart/AbstractUploader.php
index 75e6794660..7d905d938f 100644
--- a/src/Multipart/AbstractUploader.php
+++ b/src/Multipart/AbstractUploader.php
@@ -60,6 +60,8 @@ protected function getUploadCommands(callable $resultHandler)
);
$command->getHandlerList()->appendSign($resultHandler, 'mup');
$numberOfParts = $this->getNumberOfParts($this->state->getPartSize());
+////prints multiple times bc it's inside the if statement--------------------------------------------------------------------------------------------------
+// echo __METHOD__ . " | Calculating # of parts: " . $numberOfParts . "\n";
if (isset($numberOfParts) && $partNumber > $numberOfParts) {
throw new $this->config['exception_class'](
$this->state,
@@ -87,6 +89,8 @@ protected function getUploadCommands(callable $resultHandler)
$this->source->read($this->state->getPartSize());
}
}
+//prints near the end of the handleResult print statements? Maybe the thing about "Or do we just create parts til we reach the end of the file?"-------------------------------------------------------------------------------------------------------------
+// print "AbstractUploader Number of parts: " . $numberOfParts . "\n";
}
/**
@@ -147,4 +151,7 @@ protected function getNumberOfParts($partSize)
}
return null;
}
+
}
+
+
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 4108c4f13b..aa9334d902 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -97,6 +97,7 @@ public function markPartAsUploaded($partNumber, array $partData = [])
*/
public function hasPartBeenUploaded($partNumber)
{
+// echo __METHOD__ . " | checking if uploaded: " . $partNumber . "\n";
return isset($this->uploadedParts[$partNumber]);
}
@@ -108,7 +109,6 @@ public function hasPartBeenUploaded($partNumber)
public function getUploadedParts()
{
ksort($this->uploadedParts);
-
return $this->uploadedParts;
}
@@ -118,6 +118,14 @@ public function getUploadedParts()
* @param int $status Status is an integer code defined by the constants
* CREATED, INITIATED, and COMPLETED on this class.
*/
+
+// public function updateProgressBar($contentLength)
+// {
+//// echo "part size " . $this->partSize . "\n";
+//
+// echo "total uploaded: " . ($this->uploadedBytes += $contentLength) . "\n";
+// return array_shift($this->progressBar);
+// }
public function setStatus($status)
{
$this->status = $status;
diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php
index ae47d7e5fd..eefe472d9b 100644
--- a/src/S3/MultipartUploader.php
+++ b/src/S3/MultipartUploader.php
@@ -19,6 +19,18 @@ class MultipartUploader extends AbstractUploader
const PART_MIN_SIZE = 5242880;
const PART_MAX_SIZE = 5368709120;
const PART_MAX_NUM = 10000;
+ private $uploadedBytes = 0;
+ private $progressBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"
+ ];
/**
* Creates a multipart upload for an S3 object.
@@ -70,6 +82,13 @@ public function __construct(
'key' => null,
'exception_class' => S3MultipartUploadException::class,
]);
+ $totalSize = $this->source->getSize();
+ $this->progressThresholds = [];
+ for ($i=1;$i<=8;$i++) {
+ $this->progressThresholds []= round($totalSize*($i/8));
+ }
+ print_r($this->progressThresholds);
+ echo array_shift($this->progressBar);
}
protected function loadUploadWorkflowInfo()
@@ -134,10 +153,24 @@ protected function createPart($seekable, $number)
}
$data['ContentLength'] = $contentLength;
+// echo $data['ContentLength'];
+ $this->uploadedBytes += $contentLength;
+ $this->displayProgress($contentLength);
return $data;
}
+ protected function displayProgress($length)
+ {
+ $threshold = $this->progressThresholds;
+
+ if (!empty($threshold) and !empty($this->progressBar) and $this->uploadedBytes >= $threshold[0]) {
+ echo $this->uploadedBytes . " is larger than or = to " . $threshold[0] . "\n";
+ array_shift($this->progressThresholds);
+ echo array_shift($this->progressBar);
+ }
+ }
+
protected function extractETag(ResultInterface $result)
{
return $result['ETag'];
diff --git a/src/S3/MultipartUploadingTrait.php b/src/S3/MultipartUploadingTrait.php
index baccf58c51..2cc8f23612 100644
--- a/src/S3/MultipartUploadingTrait.php
+++ b/src/S3/MultipartUploadingTrait.php
@@ -55,8 +55,15 @@ protected function handleResult(CommandInterface $command, ResultInterface $resu
'PartNumber' => $command['PartNumber'],
'ETag' => $this->extractETag($result),
]);
+// $progressResult = $this->getState()->updateProgressBar($command["ContentLength"]);
+// echo $progressResult;
}
+ protected function displayProgress(CommandInterface $command)
+ {
+ $progressResult = $this->getState()->updateProgressBar($command["ContentLength"]);
+ return $progressResult;
+ }
abstract protected function extractETag(ResultInterface $result);
protected function getCompleteParams()
diff --git a/src/S3/ObjectUploader.php b/src/S3/ObjectUploader.php
index 4bbc583a71..2cbf018226 100644
--- a/src/S3/ObjectUploader.php
+++ b/src/S3/ObjectUploader.php
@@ -63,6 +63,12 @@ public function __construct(
// Handle "add_content_md5" option.
$this->addContentMD5 = isset($options['add_content_md5'])
&& $options['add_content_md5'] === true;
+ /*$totalSize = $this->body->getSize();
+ $this->progressThresholds = [];
+ for ($i=1;$i<=8;$i++) {
+ $this->progressThresholds []= round($totalSize*($i/8));
+ }
+ print_r($this->progressThresholds);*/
}
/**
@@ -74,6 +80,7 @@ public function promise()
$mup_threshold = $this->options['mup_threshold'];
if ($this->requiresMultipart($this->body, $mup_threshold)) {
// Perform a multipart upload.
+ echo "Total bytes: " . $this->body->getSize() . "\n";
return (new MultipartUploader($this->client, $this->body, [
'bucket' => $this->bucket,
'key' => $this->key,
@@ -92,12 +99,19 @@ public function promise()
if (is_callable($this->options['before_upload'])) {
$this->options['before_upload']($command);
}
- return $this->client->executeAsync($command);
+// return $this->client->executeAsync($command);
+ $test = $this->client->executeAsync($command);
+ return $test;
+
}
public function upload()
{
- return $this->promise()->wait();
+ $result = $this->promise()->wait();
+// if ($result["@metadata"]["statusCode"] == '200') {
+// echo "|====================| 100.0%\nTransfer complete!\n";
+// }
+ return $result;
}
/**
From 51f757452069638583ec5eb7561298c2045598eb Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 17 Apr 2023 12:14:29 -0400
Subject: [PATCH 02/31] update displayProgress method
---
src/S3/MultipartUploader.php | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php
index eefe472d9b..a4e77603b1 100644
--- a/src/S3/MultipartUploader.php
+++ b/src/S3/MultipartUploader.php
@@ -153,14 +153,13 @@ protected function createPart($seekable, $number)
}
$data['ContentLength'] = $contentLength;
-// echo $data['ContentLength'];
$this->uploadedBytes += $contentLength;
- $this->displayProgress($contentLength);
+ $this->displayProgress();
return $data;
}
- protected function displayProgress($length)
+ protected function displayProgress()
{
$threshold = $this->progressThresholds;
From 5cb12d5128908c9018b898dc54df0dc5c507f7fa Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 17 Apr 2023 12:47:56 -0400
Subject: [PATCH 03/31] removed comments
---
src/Multipart/AbstractUploader.php | 2 --
src/Multipart/UploadState.php | 7 -------
src/S3/MultipartUploader.php | 2 --
src/S3/MultipartUploadingTrait.php | 7 -------
src/S3/ObjectUploader.php | 10 ----------
5 files changed, 28 deletions(-)
diff --git a/src/Multipart/AbstractUploader.php b/src/Multipart/AbstractUploader.php
index 7d905d938f..848bb6b5ca 100644
--- a/src/Multipart/AbstractUploader.php
+++ b/src/Multipart/AbstractUploader.php
@@ -60,8 +60,6 @@ protected function getUploadCommands(callable $resultHandler)
);
$command->getHandlerList()->appendSign($resultHandler, 'mup');
$numberOfParts = $this->getNumberOfParts($this->state->getPartSize());
-////prints multiple times bc it's inside the if statement--------------------------------------------------------------------------------------------------
-// echo __METHOD__ . " | Calculating # of parts: " . $numberOfParts . "\n";
if (isset($numberOfParts) && $partNumber > $numberOfParts) {
throw new $this->config['exception_class'](
$this->state,
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index aa9334d902..105ffab24b 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -119,13 +119,6 @@ public function getUploadedParts()
* CREATED, INITIATED, and COMPLETED on this class.
*/
-// public function updateProgressBar($contentLength)
-// {
-//// echo "part size " . $this->partSize . "\n";
-//
-// echo "total uploaded: " . ($this->uploadedBytes += $contentLength) . "\n";
-// return array_shift($this->progressBar);
-// }
public function setStatus($status)
{
$this->status = $status;
diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php
index a4e77603b1..90469a1000 100644
--- a/src/S3/MultipartUploader.php
+++ b/src/S3/MultipartUploader.php
@@ -87,7 +87,6 @@ public function __construct(
for ($i=1;$i<=8;$i++) {
$this->progressThresholds []= round($totalSize*($i/8));
}
- print_r($this->progressThresholds);
echo array_shift($this->progressBar);
}
@@ -164,7 +163,6 @@ protected function displayProgress()
$threshold = $this->progressThresholds;
if (!empty($threshold) and !empty($this->progressBar) and $this->uploadedBytes >= $threshold[0]) {
- echo $this->uploadedBytes . " is larger than or = to " . $threshold[0] . "\n";
array_shift($this->progressThresholds);
echo array_shift($this->progressBar);
}
diff --git a/src/S3/MultipartUploadingTrait.php b/src/S3/MultipartUploadingTrait.php
index 2cc8f23612..baccf58c51 100644
--- a/src/S3/MultipartUploadingTrait.php
+++ b/src/S3/MultipartUploadingTrait.php
@@ -55,15 +55,8 @@ protected function handleResult(CommandInterface $command, ResultInterface $resu
'PartNumber' => $command['PartNumber'],
'ETag' => $this->extractETag($result),
]);
-// $progressResult = $this->getState()->updateProgressBar($command["ContentLength"]);
-// echo $progressResult;
}
- protected function displayProgress(CommandInterface $command)
- {
- $progressResult = $this->getState()->updateProgressBar($command["ContentLength"]);
- return $progressResult;
- }
abstract protected function extractETag(ResultInterface $result);
protected function getCompleteParams()
diff --git a/src/S3/ObjectUploader.php b/src/S3/ObjectUploader.php
index 2cbf018226..2b899b97a3 100644
--- a/src/S3/ObjectUploader.php
+++ b/src/S3/ObjectUploader.php
@@ -63,12 +63,6 @@ public function __construct(
// Handle "add_content_md5" option.
$this->addContentMD5 = isset($options['add_content_md5'])
&& $options['add_content_md5'] === true;
- /*$totalSize = $this->body->getSize();
- $this->progressThresholds = [];
- for ($i=1;$i<=8;$i++) {
- $this->progressThresholds []= round($totalSize*($i/8));
- }
- print_r($this->progressThresholds);*/
}
/**
@@ -80,7 +74,6 @@ public function promise()
$mup_threshold = $this->options['mup_threshold'];
if ($this->requiresMultipart($this->body, $mup_threshold)) {
// Perform a multipart upload.
- echo "Total bytes: " . $this->body->getSize() . "\n";
return (new MultipartUploader($this->client, $this->body, [
'bucket' => $this->bucket,
'key' => $this->key,
@@ -108,9 +101,6 @@ public function promise()
public function upload()
{
$result = $this->promise()->wait();
-// if ($result["@metadata"]["statusCode"] == '200') {
-// echo "|====================| 100.0%\nTransfer complete!\n";
-// }
return $result;
}
From ebd06c4da9f2fcfd1b877da0fe6238b50311e670 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Tue, 18 Apr 2023 23:39:01 -0400
Subject: [PATCH 04/31] added ability to display progressBar for
MultipartUpload and MultipartCopy
---
src/Multipart/AbstractUploader.php | 2 --
src/Multipart/UploadState.php | 29 +++++++++++++++++++++++++++
src/S3/MultipartCopy.php | 2 ++
src/S3/MultipartUploader.php | 32 ++----------------------------
src/S3/MultipartUploadingTrait.php | 14 +++++++++++++
src/S3/ObjectUploader.php | 8 ++------
6 files changed, 49 insertions(+), 38 deletions(-)
diff --git a/src/Multipart/AbstractUploader.php b/src/Multipart/AbstractUploader.php
index 848bb6b5ca..dc1724f7a0 100644
--- a/src/Multipart/AbstractUploader.php
+++ b/src/Multipart/AbstractUploader.php
@@ -87,8 +87,6 @@ protected function getUploadCommands(callable $resultHandler)
$this->source->read($this->state->getPartSize());
}
}
-//prints near the end of the handleResult print statements? Maybe the thing about "Or do we just create parts til we reach the end of the file?"-------------------------------------------------------------------------------------------------------------
-// print "AbstractUploader Number of parts: " . $numberOfParts . "\n";
}
/**
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 105ffab24b..807a685521 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -13,6 +13,18 @@ class UploadState
const INITIATED = 1;
const COMPLETED = 2;
+ protected $progressBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"
+ ];
+
/** @var array Params used to identity the upload. */
private $id;
@@ -31,6 +43,7 @@ class UploadState
public function __construct(array $id)
{
$this->id = $id;
+ echo array_shift($this->progressBar);
}
/**
@@ -76,6 +89,22 @@ public function setPartSize($partSize)
$this->partSize = $partSize;
}
+ public function setProgressThresholds($thresholds)
+ {
+ $this->progressThresholds = $thresholds;
+ }
+
+ public function displayProgress($totalUploaded)
+ {
+ while (!empty($this->progressThresholds)
+ && !empty($this->progressBar)
+ && $totalUploaded >= $this->progressThresholds[0])
+ {
+ array_shift($this->progressThresholds);
+ echo array_shift($this->progressBar);
+ }
+ }
+
/**
* Marks a part as being uploaded.
*
diff --git a/src/S3/MultipartCopy.php b/src/S3/MultipartCopy.php
index 5b26dea79e..7383910d3b 100644
--- a/src/S3/MultipartCopy.php
+++ b/src/S3/MultipartCopy.php
@@ -75,6 +75,8 @@ public function __construct(
$client,
array_change_key_case($config) + ['source_metadata' => null]
);
+
+ $this->createProgressThresholds($this->sourceMetadata["ContentLength"]);
}
/**
diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php
index 90469a1000..0c9a49bf1c 100644
--- a/src/S3/MultipartUploader.php
+++ b/src/S3/MultipartUploader.php
@@ -19,18 +19,6 @@ class MultipartUploader extends AbstractUploader
const PART_MIN_SIZE = 5242880;
const PART_MAX_SIZE = 5368709120;
const PART_MAX_NUM = 10000;
- private $uploadedBytes = 0;
- private $progressBar = [
- "Transfer initiated...\n| | 0.0%\n",
- "|== | 12.5%\n",
- "|===== | 25.0%\n",
- "|======= | 37.5%\n",
- "|========== | 50.0%\n",
- "|============ | 62.5%\n",
- "|=============== | 75.0%\n",
- "|================= | 87.5%\n",
- "|====================| 100.0%\nTransfer complete!\n"
- ];
/**
* Creates a multipart upload for an S3 object.
@@ -82,12 +70,8 @@ public function __construct(
'key' => null,
'exception_class' => S3MultipartUploadException::class,
]);
- $totalSize = $this->source->getSize();
- $this->progressThresholds = [];
- for ($i=1;$i<=8;$i++) {
- $this->progressThresholds []= round($totalSize*($i/8));
- }
- echo array_shift($this->progressBar);
+
+ $this->createProgressThresholds($this->source->getSize());
}
protected function loadUploadWorkflowInfo()
@@ -152,22 +136,10 @@ protected function createPart($seekable, $number)
}
$data['ContentLength'] = $contentLength;
- $this->uploadedBytes += $contentLength;
- $this->displayProgress();
return $data;
}
- protected function displayProgress()
- {
- $threshold = $this->progressThresholds;
-
- if (!empty($threshold) and !empty($this->progressBar) and $this->uploadedBytes >= $threshold[0]) {
- array_shift($this->progressThresholds);
- echo array_shift($this->progressBar);
- }
- }
-
protected function extractETag(ResultInterface $result)
{
return $result['ETag'];
diff --git a/src/S3/MultipartUploadingTrait.php b/src/S3/MultipartUploadingTrait.php
index baccf58c51..39d9e9917d 100644
--- a/src/S3/MultipartUploadingTrait.php
+++ b/src/S3/MultipartUploadingTrait.php
@@ -7,6 +7,8 @@
trait MultipartUploadingTrait
{
+ private $uploadedBytes = 0;
+
/**
* Creates an UploadState object for a multipart upload by querying the
* service for the specified upload's information.
@@ -49,12 +51,24 @@ public static function getStateFromService(
return $state;
}
+ protected function createProgressThresholds($totalSize)
+ {
+ $this->progressThresholds = [];
+ for ($i=1;$i<=8;$i++) {
+ $this->progressThresholds []= round($totalSize*($i/8));
+ }
+ $this->getState()->setProgressThresholds($this->progressThresholds);
+ }
+
protected function handleResult(CommandInterface $command, ResultInterface $result)
{
$this->getState()->markPartAsUploaded($command['PartNumber'], [
'PartNumber' => $command['PartNumber'],
'ETag' => $this->extractETag($result),
]);
+
+ $this->uploadedBytes += $command["ContentLength"];
+ $this->getState()->displayProgress($this->uploadedBytes);
}
abstract protected function extractETag(ResultInterface $result);
diff --git a/src/S3/ObjectUploader.php b/src/S3/ObjectUploader.php
index 2b899b97a3..4bbc583a71 100644
--- a/src/S3/ObjectUploader.php
+++ b/src/S3/ObjectUploader.php
@@ -92,16 +92,12 @@ public function promise()
if (is_callable($this->options['before_upload'])) {
$this->options['before_upload']($command);
}
-// return $this->client->executeAsync($command);
- $test = $this->client->executeAsync($command);
- return $test;
-
+ return $this->client->executeAsync($command);
}
public function upload()
{
- $result = $this->promise()->wait();
- return $result;
+ return $this->promise()->wait();
}
/**
From 6dc3597ceb3cfa5ed20e5fabc39ef6ccffd93d4b Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Tue, 18 Apr 2023 23:46:43 -0400
Subject: [PATCH 05/31] Update UploadState.php
---
src/Multipart/UploadState.php | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 807a685521..179d8d1018 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -126,7 +126,6 @@ public function markPartAsUploaded($partNumber, array $partData = [])
*/
public function hasPartBeenUploaded($partNumber)
{
-// echo __METHOD__ . " | checking if uploaded: " . $partNumber . "\n";
return isset($this->uploadedParts[$partNumber]);
}
From 7f05ff1c527ca0eff59757299b3a859d7509b19a Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Wed, 19 Apr 2023 15:18:37 -0400
Subject: [PATCH 06/31] Update AbstractUploader.php
---
src/Multipart/AbstractUploader.php | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/src/Multipart/AbstractUploader.php b/src/Multipart/AbstractUploader.php
index dc1724f7a0..f0fd6e56e6 100644
--- a/src/Multipart/AbstractUploader.php
+++ b/src/Multipart/AbstractUploader.php
@@ -147,7 +147,4 @@ protected function getNumberOfParts($partSize)
}
return null;
}
-
-}
-
-
+}
\ No newline at end of file
From 657ab9a242f71760c10d4eae48a0f609e337aaf2 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Wed, 19 Apr 2023 18:17:11 -0400
Subject: [PATCH 07/31] MultipartUploader and MultipartCopy now sends total
size directly to UploadState
---
src/Multipart/UploadState.php | 8 ++++++--
src/S3/MultipartCopy.php | 4 +++-
src/S3/MultipartUploader.php | 2 +-
src/S3/MultipartUploadingTrait.php | 9 ---------
4 files changed, 10 insertions(+), 13 deletions(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 179d8d1018..0339929cfa 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -37,6 +37,8 @@ class UploadState
/** @var int Identifies the status the upload. */
private $status = self::CREATED;
+ private $progressThresholds = [];
+
/**
* @param array $id Params used to identity the upload.
*/
@@ -89,9 +91,11 @@ public function setPartSize($partSize)
$this->partSize = $partSize;
}
- public function setProgressThresholds($thresholds)
+ public function setProgressThresholds($totalSize)
{
- $this->progressThresholds = $thresholds;
+ for ($i=1;$i<=8;$i++) {
+ $this->progressThresholds []= round($totalSize*($i/8));
+ }
}
public function displayProgress($totalUploaded)
diff --git a/src/S3/MultipartCopy.php b/src/S3/MultipartCopy.php
index 7383910d3b..00e4d73b02 100644
--- a/src/S3/MultipartCopy.php
+++ b/src/S3/MultipartCopy.php
@@ -76,7 +76,9 @@ public function __construct(
array_change_key_case($config) + ['source_metadata' => null]
);
- $this->createProgressThresholds($this->sourceMetadata["ContentLength"]);
+ $this->getState()->setProgressThresholds(
+ $this->sourceMetadata["ContentLength"]
+ );
}
/**
diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php
index 0c9a49bf1c..7570fa4538 100644
--- a/src/S3/MultipartUploader.php
+++ b/src/S3/MultipartUploader.php
@@ -71,7 +71,7 @@ public function __construct(
'exception_class' => S3MultipartUploadException::class,
]);
- $this->createProgressThresholds($this->source->getSize());
+ $this->getState()->setProgressThresholds($this->source->getSize());
}
protected function loadUploadWorkflowInfo()
diff --git a/src/S3/MultipartUploadingTrait.php b/src/S3/MultipartUploadingTrait.php
index 39d9e9917d..9e1eedacc7 100644
--- a/src/S3/MultipartUploadingTrait.php
+++ b/src/S3/MultipartUploadingTrait.php
@@ -51,15 +51,6 @@ public static function getStateFromService(
return $state;
}
- protected function createProgressThresholds($totalSize)
- {
- $this->progressThresholds = [];
- for ($i=1;$i<=8;$i++) {
- $this->progressThresholds []= round($totalSize*($i/8));
- }
- $this->getState()->setProgressThresholds($this->progressThresholds);
- }
-
protected function handleResult(CommandInterface $command, ResultInterface $result)
{
$this->getState()->markPartAsUploaded($command['PartNumber'], [
From 89c95138504d102f32092bc56129c81cad93edc2 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 24 Apr 2023 15:52:39 -0400
Subject: [PATCH 08/31] Update UploadStateTest.php
---
tests/Multipart/UploadStateTest.php | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 64ac615f30..876ae94623 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -70,4 +70,17 @@ public function testSerializationWorks()
$this->assertTrue($newState->isInitiated());
$this->assertArrayHasKey('foo', $newState->getId());
}
+
+ public function testDisplayProgressPrints()
+ {
+ ob_start();
+
+ $state = new UploadState([]);
+ $state->setProgressThresholds(100);
+ $state->displayProgress(13);
+
+ $output = ob_end_clean();
+// echo $output;
+ $this->assertEquals("Transfer initiated...\n| | 0.0%\n|== | 12.5%\n", $output);
+ }
}
From d68597cc81329197531e2d72373ef6af938c374b Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 24 Apr 2023 16:42:37 -0400
Subject: [PATCH 09/31] Added assert using expectOutputString and structure for
data provider
---
tests/Multipart/UploadStateTest.php | 28 ++++++++++++++++++++--------
1 file changed, 20 insertions(+), 8 deletions(-)
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 876ae94623..2ffd26db08 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -70,17 +70,29 @@ public function testSerializationWorks()
$this->assertTrue($newState->isInitiated());
$this->assertArrayHasKey('foo', $newState->getId());
}
-
- public function testDisplayProgressPrints()
+ /**
+ * @dataProvider getDisplayProgressCases
+ */
+ public function testDisplayProgressPrints($totalSize, $totalUploaded, $progressBar)
{
- ob_start();
+// ob_start();
$state = new UploadState([]);
- $state->setProgressThresholds(100);
- $state->displayProgress(13);
+ $state->setProgressThresholds($totalSize);
+ $state->displayProgress($totalUploaded);
+//
+// $output = ob_get_contents();
+// ob_end_clean();
+// $this->assertEquals($progressBar, $output);
+ $this->expectOutputString($progressBar);
+ }
- $output = ob_end_clean();
-// echo $output;
- $this->assertEquals("Transfer initiated...\n| | 0.0%\n|== | 12.5%\n", $output);
+ public function getDisplayProgressCases()
+ {
+ return [
+ [100, 10, "Transfer initiated...\n| | 0.0%\n"],
+ [100, 0, "Transfer initiated...\n| | 0.0%\n"],
+ [100, 13, "Transfer initiated...\n| | 0.0%\n|== | 12.5%\n"]
+ ];
}
}
From 99738f88dd6ed6c17e0b2a9009265c2908224186 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Wed, 26 Apr 2023 12:47:45 -0400
Subject: [PATCH 10/31] added two tests
---
src/Multipart/UploadState.php | 5 +--
tests/Multipart/UploadStateTest.php | 55 ++++++++++++++++++++++-------
2 files changed, 46 insertions(+), 14 deletions(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 0339929cfa..d2f487711d 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -42,7 +42,7 @@ class UploadState
/**
* @param array $id Params used to identity the upload.
*/
- public function __construct(array $id)
+ public function __construct(array $id, $config=[])
{
$this->id = $id;
echo array_shift($this->progressBar);
@@ -96,6 +96,7 @@ public function setProgressThresholds($totalSize)
for ($i=1;$i<=8;$i++) {
$this->progressThresholds []= round($totalSize*($i/8));
}
+ return $this->progressThresholds;
}
public function displayProgress($totalUploaded)
@@ -175,4 +176,4 @@ public function isCompleted()
{
return $this->status === self::COMPLETED;
}
-}
+}
\ No newline at end of file
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 2ffd26db08..8886ad39c6 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -73,26 +73,57 @@ public function testSerializationWorks()
/**
* @dataProvider getDisplayProgressCases
*/
- public function testDisplayProgressPrints($totalSize, $totalUploaded, $progressBar)
- {
-// ob_start();
-
+ public function testDisplayProgressPrints(
+ $totalSize,
+ $totalUploaded,
+ $progressBar
+ ) {
$state = new UploadState([]);
$state->setProgressThresholds($totalSize);
$state->displayProgress($totalUploaded);
-//
-// $output = ob_get_contents();
-// ob_end_clean();
-// $this->assertEquals($progressBar, $output);
+
$this->expectOutputString($progressBar);
}
public function getDisplayProgressCases()
+ {
+ $progressBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"
+ ];
+ return [
+ [100000, 0, $progressBar[0]],
+ [100000, 12499, $progressBar[0]],
+ [100000, 12500, "$progressBar[0]$progressBar[1]"],
+ [100000, 100000, implode($progressBar)]
+ ];
+ }
+
+ /**
+ * @dataProvider getThresholdCases
+ */
+ public function testUploadThresholds($totalSize)
+ {
+ $state = new UploadState([]);
+ $threshold = $state->setProgressThresholds($totalSize);
+
+ $this->assertIsArray($threshold);
+ $this->assertCount(8, $threshold);
+ }
+
+ public function getThresholdCases()
{
return [
- [100, 10, "Transfer initiated...\n| | 0.0%\n"],
- [100, 0, "Transfer initiated...\n| | 0.0%\n"],
- [100, 13, "Transfer initiated...\n| | 0.0%\n|== | 12.5%\n"]
+ [0],
+ [100000],
+ [100001]
];
}
-}
+}
\ No newline at end of file
From 604df58291a03d6adb5db041ff3a11077f019587 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Wed, 26 Apr 2023 12:48:37 -0400
Subject: [PATCH 11/31] Update UploadState.php
---
src/Multipart/UploadState.php | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index d2f487711d..6786202c64 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -42,7 +42,7 @@ class UploadState
/**
* @param array $id Params used to identity the upload.
*/
- public function __construct(array $id, $config=[])
+ public function __construct(array $id)
{
$this->id = $id;
echo array_shift($this->progressBar);
From 12e328d0d2488dbdb36285eb142570d739eed31a Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Fri, 28 Apr 2023 14:55:19 -0400
Subject: [PATCH 12/31] Update UploadStateTest.php
The strings in the return statement were over the 80 char limit
---
tests/Multipart/UploadStateTest.php | 77 +++++++++++++++++++++++------
1 file changed, 63 insertions(+), 14 deletions(-)
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 8886ad39c6..4439c0ebf6 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -73,7 +73,7 @@ public function testSerializationWorks()
/**
* @dataProvider getDisplayProgressCases
*/
- public function testDisplayProgressPrints(
+ public function testDisplayProgressPrintsProgress(
$totalSize,
$totalUploaded,
$progressBar
@@ -87,22 +87,71 @@ public function testDisplayProgressPrints(
public function getDisplayProgressCases()
{
- $progressBar = [
- "Transfer initiated...\n| | 0.0%\n",
- "|== | 12.5%\n",
- "|===== | 25.0%\n",
- "|======= | 37.5%\n",
- "|========== | 50.0%\n",
- "|============ | 62.5%\n",
- "|=============== | 75.0%\n",
- "|================= | 87.5%\n",
- "|====================| 100.0%\nTransfer complete!\n"
- ];
+ $progressBar = array(
+ 0 => "Transfer initiated...\n| | 0.0%\n",
+ 12.5 => "Transfer initiated...\n| | 0.0%\n" .
+ "|== | 12.5%\n",
+ 25.0 => "Transfer initiated...\n| | 0.0%\n" .
+ "|== | 12.5%\n" .
+ "|===== | 25.0%\n",
+ 37.5 => "Transfer initiated...\n| | 0.0%\n" .
+ "|== | 12.5%\n" .
+ "|===== | 25.0%\n" .
+ "|======= | 37.5%\n",
+ 50.0 => "Transfer initiated...\n| | 0.0%\n" .
+ "|== | 12.5%\n" .
+ "|===== | 25.0%\n" .
+ "|======= | 37.5%\n" .
+ "|========== | 50.0%\n",
+ 62.5 => "Transfer initiated...\n| | 0.0%\n" .
+ "|== | 12.5%\n" .
+ "|===== | 25.0%\n" .
+ "|======= | 37.5%\n" .
+ "|========== | 50.0%\n" .
+ "|============ | 62.5%\n",
+ 75.0 => "Transfer initiated...\n| | 0.0%\n" .
+ "|== | 12.5%\n" .
+ "|===== | 25.0%\n" .
+ "|======= | 37.5%\n" .
+ "|========== | 50.0%\n" .
+ "|============ | 62.5%\n" .
+ "|=============== | 75.0%\n",
+ 87.5 => "Transfer initiated...\n| | 0.0%\n" .
+ "|== | 12.5%\n" .
+ "|===== | 25.0%\n" .
+ "|======= | 37.5%\n" .
+ "|========== | 50.0%\n" .
+ "|============ | 62.5%\n" .
+ "|=============== | 75.0%\n" .
+ "|================= | 87.5%\n",
+ 100 => "Transfer initiated...\n| | 0.0%\n" .
+ "|== | 12.5%\n" .
+ "|===== | 25.0%\n" .
+ "|======= | 37.5%\n" .
+ "|========== | 50.0%\n" .
+ "|============ | 62.5%\n" .
+ "|=============== | 75.0%\n" .
+ "|================= | 87.5%\n" .
+ "|====================| 100.0%\nTransfer complete!\n"
+ );
return [
[100000, 0, $progressBar[0]],
[100000, 12499, $progressBar[0]],
- [100000, 12500, "$progressBar[0]$progressBar[1]"],
- [100000, 100000, implode($progressBar)]
+ [100000, 12500, $progressBar[12.5]],
+ [100000, 24999, $progressBar[12.5]],
+ [100000, 25000, $progressBar[25.0]],
+ [100000, 37499, $progressBar[25.0]],
+ [100000, 37500, $progressBar[37.5]],
+ [100000, 49999, $progressBar[37.5]],
+ [100000, 50000, $progressBar[50.0]],
+ [100000, 62499, $progressBar[50.0]],
+ [100000, 62500, $progressBar[62.5]],
+ [100000, 74999, $progressBar[62.5]],
+ [100000, 75000, $progressBar[75.0]],
+ [100000, 87499, $progressBar[75.0]],
+ [100000, 87500, $progressBar[87.5]],
+ [100000, 99999, $progressBar[87.5]],
+ [100000, 100000, $progressBar[100]]
];
}
From e42ab910a92af06edb9b0af11346f311bf92e708 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 1 May 2023 12:57:19 -0400
Subject: [PATCH 13/31] Update UploadStateTest.php
---
tests/Multipart/UploadStateTest.php | 132 +++++++++++++++-------------
1 file changed, 70 insertions(+), 62 deletions(-)
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 4439c0ebf6..d85afce867 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -87,71 +87,79 @@ public function testDisplayProgressPrintsProgress(
public function getDisplayProgressCases()
{
- $progressBar = array(
- 0 => "Transfer initiated...\n| | 0.0%\n",
- 12.5 => "Transfer initiated...\n| | 0.0%\n" .
- "|== | 12.5%\n",
- 25.0 => "Transfer initiated...\n| | 0.0%\n" .
- "|== | 12.5%\n" .
- "|===== | 25.0%\n",
- 37.5 => "Transfer initiated...\n| | 0.0%\n" .
- "|== | 12.5%\n" .
- "|===== | 25.0%\n" .
- "|======= | 37.5%\n",
- 50.0 => "Transfer initiated...\n| | 0.0%\n" .
- "|== | 12.5%\n" .
- "|===== | 25.0%\n" .
- "|======= | 37.5%\n" .
- "|========== | 50.0%\n",
- 62.5 => "Transfer initiated...\n| | 0.0%\n" .
- "|== | 12.5%\n" .
- "|===== | 25.0%\n" .
- "|======= | 37.5%\n" .
- "|========== | 50.0%\n" .
- "|============ | 62.5%\n",
- 75.0 => "Transfer initiated...\n| | 0.0%\n" .
- "|== | 12.5%\n" .
- "|===== | 25.0%\n" .
- "|======= | 37.5%\n" .
- "|========== | 50.0%\n" .
- "|============ | 62.5%\n" .
- "|=============== | 75.0%\n",
- 87.5 => "Transfer initiated...\n| | 0.0%\n" .
- "|== | 12.5%\n" .
- "|===== | 25.0%\n" .
- "|======= | 37.5%\n" .
- "|========== | 50.0%\n" .
- "|============ | 62.5%\n" .
- "|=============== | 75.0%\n" .
- "|================= | 87.5%\n",
- 100 => "Transfer initiated...\n| | 0.0%\n" .
- "|== | 12.5%\n" .
- "|===== | 25.0%\n" .
- "|======= | 37.5%\n" .
- "|========== | 50.0%\n" .
- "|============ | 62.5%\n" .
- "|=============== | 75.0%\n" .
- "|================= | 87.5%\n" .
- "|====================| 100.0%\nTransfer complete!\n"
- );
+ $progressBar = ["Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"];
return [
[100000, 0, $progressBar[0]],
[100000, 12499, $progressBar[0]],
- [100000, 12500, $progressBar[12.5]],
- [100000, 24999, $progressBar[12.5]],
- [100000, 25000, $progressBar[25.0]],
- [100000, 37499, $progressBar[25.0]],
- [100000, 37500, $progressBar[37.5]],
- [100000, 49999, $progressBar[37.5]],
- [100000, 50000, $progressBar[50.0]],
- [100000, 62499, $progressBar[50.0]],
- [100000, 62500, $progressBar[62.5]],
- [100000, 74999, $progressBar[62.5]],
- [100000, 75000, $progressBar[75.0]],
- [100000, 87499, $progressBar[75.0]],
- [100000, 87500, $progressBar[87.5]],
- [100000, 99999, $progressBar[87.5]],
- [100000, 100000, $progressBar[100]]
+ [100000, 12500, "{$progressBar[0]}{$progressBar[1]}"],
+ [100000, 24999, "{$progressBar[0]}{$progressBar[1]}"],
+ [100000, 25000, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"],
+ [100000, 37499, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"],
+ [
+ 100000,
+ 37500,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}"
+ ],
+ [
+ 100000,
+ 49999,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}"
+ ],
+ [
+ 100000,
+ 50000,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}"
+ ],
+ [
+ 100000,
+ 62499,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}"
+ ],
+ [
+ 100000,
+ 62500,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+ "{$progressBar[5]}"
+ ],
+ [
+ 100000,
+ 74999,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+ "{$progressBar[5]}"
+ ],
+ [
+ 100000,
+ 75000,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+ "{$progressBar[5]}{$progressBar[6]}"
+ ],
+ [
+ 100000,
+ 87499,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+ "{$progressBar[5]}{$progressBar[6]}"
+ ],
+ [
+ 100000,
+ 87500,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+ "{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}"
+ ],
+ [
+ 100000,
+ 99999,
+ "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+ "{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}"
+ ],
+ [100000, 100000, implode($progressBar)]
];
}
From 470627a6309093098e77e5b17b926bf1117ad4c8 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 1 May 2023 15:20:04 -0400
Subject: [PATCH 14/31] Combine progressThresholds and progressBar into one
array
(1) Set progressThresholds[0] so that I could use array_combine on equal sized arrays
(2) I redefined progressBar in setProgressThresholds after the threshold bar is created. progressBar includes the thresholds as keys and the bar as values
(3) Removed the array_shift in the construct and now that logic is done by displayProgress
(4) Changed assertCount to check for 9 elements instead of 8
---
src/Multipart/UploadState.php | 3 ++-
tests/Multipart/UploadStateTest.php | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 6786202c64..8094cda24e 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -45,7 +45,6 @@ class UploadState
public function __construct(array $id)
{
$this->id = $id;
- echo array_shift($this->progressBar);
}
/**
@@ -93,9 +92,11 @@ public function setPartSize($partSize)
public function setProgressThresholds($totalSize)
{
+ $this->progressThresholds[0] = 0;
for ($i=1;$i<=8;$i++) {
$this->progressThresholds []= round($totalSize*($i/8));
}
+ $this->progressBar = array_combine($this->progressThresholds, $this->progressBar);
return $this->progressThresholds;
}
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index d85afce867..85cd821f36 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -172,7 +172,7 @@ public function testUploadThresholds($totalSize)
$threshold = $state->setProgressThresholds($totalSize);
$this->assertIsArray($threshold);
- $this->assertCount(8, $threshold);
+ $this->assertCount(9, $threshold);
}
public function getThresholdCases()
From 8cfe7e992c3b473da32cb744c290cb7eb4ebd051 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 1 May 2023 16:50:26 -0400
Subject: [PATCH 15/31] Updated displayProgress to use progressBar
---
src/Multipart/UploadState.php | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 8094cda24e..bc19cfe139 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -103,11 +103,10 @@ public function setProgressThresholds($totalSize)
public function displayProgress($totalUploaded)
{
while (!empty($this->progressThresholds)
- && !empty($this->progressBar)
&& $totalUploaded >= $this->progressThresholds[0])
{
+ echo $this->progressBar[$this->progressThresholds[0]];
array_shift($this->progressThresholds);
- echo array_shift($this->progressBar);
}
}
From 9ade47371ca9b1f468e073aec8515a6b5340e17c Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Tue, 2 May 2023 12:54:36 -0400
Subject: [PATCH 16/31] Update UploadState.php
---
src/Multipart/UploadState.php | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index bc19cfe139..569b3b1a63 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -102,11 +102,11 @@ public function setProgressThresholds($totalSize)
public function displayProgress($totalUploaded)
{
- while (!empty($this->progressThresholds)
- && $totalUploaded >= $this->progressThresholds[0])
+ while (!empty($this->progressBar)
+ && $totalUploaded >= array_key_first($this->progressBar))
{
- echo $this->progressBar[$this->progressThresholds[0]];
- array_shift($this->progressThresholds);
+ echo $this->progressBar[array_key_first($this->progressBar)];
+ unset($this->progressBar[array_key_first($this->progressBar)]);
}
}
From 74e7938b01bfb3923909b4eeb10ee9148e549cfa Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Tue, 2 May 2023 22:06:50 -0400
Subject: [PATCH 17/31] added exceptions for non-int arguments
---
src/Multipart/UploadState.php | 8 ++++++++
tests/Multipart/UploadStateTest.php | 18 ++++++++++++++++++
2 files changed, 26 insertions(+)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 569b3b1a63..bb67749019 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -92,6 +92,10 @@ public function setPartSize($partSize)
public function setProgressThresholds($totalSize)
{
+ if(!is_numeric($totalSize)) {
+ throw new \InvalidArgumentException('The total size of the upload must be an int.');
+ }
+
$this->progressThresholds[0] = 0;
for ($i=1;$i<=8;$i++) {
$this->progressThresholds []= round($totalSize*($i/8));
@@ -102,6 +106,10 @@ public function setProgressThresholds($totalSize)
public function displayProgress($totalUploaded)
{
+ if(!is_numeric($totalUploaded)) {
+ throw new \InvalidArgumentException('The size of the bytes being uploaded must be an int.');
+ }
+
while (!empty($this->progressBar)
&& $totalUploaded >= array_key_first($this->progressBar))
{
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 85cd821f36..40feaf9b4d 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -183,4 +183,22 @@ public function getThresholdCases()
[100001]
];
}
+
+ public function testSetProgressThresholdsThrowsException()
+ {
+ $state = new UploadState([]);
+ $this->expectExceptionMessage('The total size of the upload must be an int.');
+ $this->expectException(\InvalidArgumentException::class);
+
+ $state->setProgressThresholds('');
+ }
+
+ public function testDisplayProgressThrowsException()
+ {
+ $state = new UploadState([]);
+ $this->expectExceptionMessage('The size of the bytes being uploaded must be an int.');
+ $this->expectException(\InvalidArgumentException::class);
+
+ $state->displayProgress('');
+ }
}
\ No newline at end of file
From 5eded672182389f9d000c661279926a16880db6e Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Wed, 3 May 2023 16:04:04 -0400
Subject: [PATCH 18/31] Added unit tests for failed upload and type checking
(1) testFailedUploadPrintsPartialProgressBar - upload fails at 25% then throws exception
(2) testSetProgressThresholdsThrowsException - checks non-ints
(3) testDisplayProgressThrowsException - checks non-ints
---
src/Multipart/UploadState.php | 4 +--
tests/Multipart/UploadStateTest.php | 24 +++++++++++++---
tests/S3/MultipartUploaderTest.php | 44 +++++++++++++++++++++++++++++
3 files changed, 66 insertions(+), 6 deletions(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index bb67749019..0d7ddd5aee 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -92,7 +92,7 @@ public function setPartSize($partSize)
public function setProgressThresholds($totalSize)
{
- if(!is_numeric($totalSize)) {
+ if(!is_int($totalSize)) {
throw new \InvalidArgumentException('The total size of the upload must be an int.');
}
@@ -106,7 +106,7 @@ public function setProgressThresholds($totalSize)
public function displayProgress($totalUploaded)
{
- if(!is_numeric($totalUploaded)) {
+ if(!is_int($totalUploaded)) {
throw new \InvalidArgumentException('The size of the bytes being uploaded must be an int.');
}
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 40feaf9b4d..70565951a7 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -184,21 +184,37 @@ public function getThresholdCases()
];
}
- public function testSetProgressThresholdsThrowsException()
+ /**
+ * @dataProvider getInvalidIntCases
+ */
+ public function testSetProgressThresholdsThrowsException($totalSize)
{
$state = new UploadState([]);
$this->expectExceptionMessage('The total size of the upload must be an int.');
$this->expectException(\InvalidArgumentException::class);
- $state->setProgressThresholds('');
+ $state->setProgressThresholds($totalSize);
}
- public function testDisplayProgressThrowsException()
+ /**
+ * @dataProvider getInvalidIntCases
+ */
+ public function testDisplayProgressThrowsException($totalUploaded)
{
$state = new UploadState([]);
$this->expectExceptionMessage('The size of the bytes being uploaded must be an int.');
$this->expectException(\InvalidArgumentException::class);
- $state->displayProgress('');
+ $state->displayProgress($totalUploaded);
+ }
+
+ public function getInvalidIntCases()
+ {
+ return [
+ [''],
+ [null],
+ ['1234'],
+ ['aws'],
+ ];
}
}
\ No newline at end of file
diff --git a/tests/S3/MultipartUploaderTest.php b/tests/S3/MultipartUploaderTest.php
index c5beefae92..3f93c826c9 100644
--- a/tests/S3/MultipartUploaderTest.php
+++ b/tests/S3/MultipartUploaderTest.php
@@ -316,4 +316,48 @@ public function testAppliesAmbiguousSuccessParsing()
);
$uploader->upload();
}
+
+ public function testFailedUploadPrintsPartialProgressBar($counterLimit)
+ {
+ $partialBar = [ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n"];
+ $this->expectOutputString("{$partialBar[0]}{$partialBar[1]}{$partialBar[2]}");
+
+ $this->expectExceptionMessage("An exception occurred while uploading parts to a multipart upload");
+ $this->expectException(\Aws\S3\Exception\S3MultipartUploadException::class);
+ $counter = 0;
+
+ $httpHandler = function ($request, array $options) use (&$counter) {
+ if ($counter < 4) {
+ $body = "baz";
+ } else {
+ $body = "\n\n\n";
+ }
+ $counter++;
+
+ return Promise\Create::promiseFor(
+ new Psr7\Response(200, [], $body)
+ );
+ };
+
+ $s3 = new S3Client([
+ 'version' => 'latest',
+ 'region' => 'us-east-1',
+ 'http_handler' => $httpHandler
+ ]);
+
+ $data = str_repeat('.', 50 * self::MB);
+ $source = Psr7\Utils::streamFor($data);
+
+ $uploader = new MultipartUploader(
+ $s3,
+ $source,
+ [
+ 'bucket' => 'test-bucket',
+ 'key' => 'test-key'
+ ]
+ );
+ $uploader->upload();
+ }
}
From 735c9c4758056e3fbe53dcab821e0e34960f67cb Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Fri, 5 May 2023 15:41:40 -0400
Subject: [PATCH 19/31] Added config option
---
src/Multipart/UploadState.php | 11 +++++++++--
tests/Multipart/UploadStateTest.php | 16 ++++++++++++++--
2 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index 0d7ddd5aee..b3fe0628dc 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -39,12 +39,19 @@ class UploadState
private $progressThresholds = [];
+ private $displayUploadProgress;
+
/**
* @param array $id Params used to identity the upload.
*/
- public function __construct(array $id)
+ public function __construct(array $id, array $config = [])
{
$this->id = $id;
+ if (isset($config['track_upload']) && $config['track_upload']) {
+ $this->displayUploadProgress = true;
+ } else {
+ $this->displayUploadProgress = false;
+ }
}
/**
@@ -110,7 +117,7 @@ public function displayProgress($totalUploaded)
throw new \InvalidArgumentException('The size of the bytes being uploaded must be an int.');
}
- while (!empty($this->progressBar)
+ while (!empty($this->progressBar && $this->displayUploadProgress)
&& $totalUploaded >= array_key_first($this->progressBar))
{
echo $this->progressBar[array_key_first($this->progressBar)];
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 70565951a7..525e5c5d30 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -70,6 +70,16 @@ public function testSerializationWorks()
$this->assertTrue($newState->isInitiated());
$this->assertArrayHasKey('foo', $newState->getId());
}
+
+ public function testEmptyUploadStateOutputWithConfigFalse()
+ {
+ $config['track_upload'] = false;
+ $state = new UploadState([], $config);
+ $state->setProgressThresholds(100);
+ $state->displayProgress(13);
+ $this->expectOutputString('');
+ }
+
/**
* @dataProvider getDisplayProgressCases
*/
@@ -78,7 +88,8 @@ public function testDisplayProgressPrintsProgress(
$totalUploaded,
$progressBar
) {
- $state = new UploadState([]);
+ $config['track_upload'] = true;
+ $state = new UploadState([], $config);
$state->setProgressThresholds($totalSize);
$state->displayProgress($totalUploaded);
@@ -168,7 +179,8 @@ public function getDisplayProgressCases()
*/
public function testUploadThresholds($totalSize)
{
- $state = new UploadState([]);
+ $config['track_upload'] = true;
+ $state = new UploadState([], $config);
$threshold = $state->setProgressThresholds($totalSize);
$this->assertIsArray($threshold);
From cff396ae8d78d43407e2596281be7013df2fcd9c Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Wed, 10 May 2023 14:53:28 -0400
Subject: [PATCH 20/31] added config option to multipartUploader and
multipartCopy
- If config option is true, the total size is sent to upload state and a threshold array is created. DisplayProgress now relies on there being a threshold array, if there is no threshold array (if config = no), nothing prints.
- the progress bar for multipart copy prints halfway for some reason, but the item is still copied (i'll fix this in the next commit)
---
src/Multipart/UploadState.php | 12 ++++--------
src/S3/MultipartCopy.php | 8 +++++---
src/S3/MultipartUploader.php | 5 +++--
tests/Multipart/UploadStateTest.php | 4 ++--
tests/S3/MultipartUploaderTest.php | 5 +++--
5 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/src/Multipart/UploadState.php b/src/Multipart/UploadState.php
index b3fe0628dc..94c1354aaf 100644
--- a/src/Multipart/UploadState.php
+++ b/src/Multipart/UploadState.php
@@ -39,19 +39,14 @@ class UploadState
private $progressThresholds = [];
- private $displayUploadProgress;
+// private $displayUploadProgress;
/**
* @param array $id Params used to identity the upload.
*/
- public function __construct(array $id, array $config = [])
+ public function __construct(array $id)
{
$this->id = $id;
- if (isset($config['track_upload']) && $config['track_upload']) {
- $this->displayUploadProgress = true;
- } else {
- $this->displayUploadProgress = false;
- }
}
/**
@@ -117,7 +112,8 @@ public function displayProgress($totalUploaded)
throw new \InvalidArgumentException('The size of the bytes being uploaded must be an int.');
}
- while (!empty($this->progressBar && $this->displayUploadProgress)
+ while ($this->progressThresholds
+ && !empty($this->progressBar)
&& $totalUploaded >= array_key_first($this->progressBar))
{
echo $this->progressBar[array_key_first($this->progressBar)];
diff --git a/src/S3/MultipartCopy.php b/src/S3/MultipartCopy.php
index 00e4d73b02..c79a135e75 100644
--- a/src/S3/MultipartCopy.php
+++ b/src/S3/MultipartCopy.php
@@ -76,9 +76,11 @@ public function __construct(
array_change_key_case($config) + ['source_metadata' => null]
);
- $this->getState()->setProgressThresholds(
- $this->sourceMetadata["ContentLength"]
- );
+ if (isset($config['track_upload']) && $config['track_upload']) {
+ $this->getState()->setProgressThresholds(
+ $this->sourceMetadata["ContentLength"]
+ );
+ }
}
/**
diff --git a/src/S3/MultipartUploader.php b/src/S3/MultipartUploader.php
index 7570fa4538..ab7426607e 100644
--- a/src/S3/MultipartUploader.php
+++ b/src/S3/MultipartUploader.php
@@ -70,8 +70,9 @@ public function __construct(
'key' => null,
'exception_class' => S3MultipartUploadException::class,
]);
-
- $this->getState()->setProgressThresholds($this->source->getSize());
+ if (isset($config['track_upload']) && $config['track_upload']) {
+ $this->getState()->setProgressThresholds($this->source->getSize());
+ }
}
protected function loadUploadWorkflowInfo()
diff --git a/tests/Multipart/UploadStateTest.php b/tests/Multipart/UploadStateTest.php
index 525e5c5d30..27a359e854 100644
--- a/tests/Multipart/UploadStateTest.php
+++ b/tests/Multipart/UploadStateTest.php
@@ -89,7 +89,7 @@ public function testDisplayProgressPrintsProgress(
$progressBar
) {
$config['track_upload'] = true;
- $state = new UploadState([], $config);
+ $state = new UploadState([]);
$state->setProgressThresholds($totalSize);
$state->displayProgress($totalUploaded);
@@ -180,7 +180,7 @@ public function getDisplayProgressCases()
public function testUploadThresholds($totalSize)
{
$config['track_upload'] = true;
- $state = new UploadState([], $config);
+ $state = new UploadState([]);
$threshold = $state->setProgressThresholds($totalSize);
$this->assertIsArray($threshold);
diff --git a/tests/S3/MultipartUploaderTest.php b/tests/S3/MultipartUploaderTest.php
index 3f93c826c9..d5a53a40be 100644
--- a/tests/S3/MultipartUploaderTest.php
+++ b/tests/S3/MultipartUploaderTest.php
@@ -317,7 +317,7 @@ public function testAppliesAmbiguousSuccessParsing()
$uploader->upload();
}
- public function testFailedUploadPrintsPartialProgressBar($counterLimit)
+ public function testFailedUploadPrintsPartialProgressBar()
{
$partialBar = [ "Transfer initiated...\n| | 0.0%\n",
"|== | 12.5%\n",
@@ -355,7 +355,8 @@ public function testFailedUploadPrintsPartialProgressBar($counterLimit)
$source,
[
'bucket' => 'test-bucket',
- 'key' => 'test-key'
+ 'key' => 'test-key',
+ 'track_upload' => 'true'
]
);
$uploader->upload();
From 02b99aaccefa6677564c1fcd03560f11e5961a69 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Wed, 17 May 2023 15:47:25 -0400
Subject: [PATCH 21/31] copied upload classes
---
src/Multipart/AbstractDownloadManager.php | 321 ++++++++++++++++++++++
src/Multipart/AbstractDownloader.php | 150 ++++++++++
src/S3/MultipartDownloader.php | 178 ++++++++++++
src/S3/MultipartDownloadingTrait.php | 137 +++++++++
4 files changed, 786 insertions(+)
create mode 100644 src/Multipart/AbstractDownloadManager.php
create mode 100644 src/Multipart/AbstractDownloader.php
create mode 100644 src/S3/MultipartDownloader.php
create mode 100644 src/S3/MultipartDownloadingTrait.php
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
new file mode 100644
index 0000000000..6d47ae77d6
--- /dev/null
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -0,0 +1,321 @@
+ null,
+ 'state' => null,
+ 'concurrency' => self::DEFAULT_CONCURRENCY,
+ 'prepare_data_source' => null,
+ 'before_initiate' => null,
+ 'before_upload' => null,
+ 'before_complete' => null,
+ 'exception_class' => 'Aws\Exception\MultipartUploadException',
+ ];
+
+ /** @var Client Client used for the upload. */
+ protected $client;
+
+ /** @var array Configuration used to perform the upload. */
+ protected $config;
+
+ /** @var array Service-specific information about the upload workflow. */
+ protected $info;
+
+ /** @var PromiseInterface Promise that represents the multipart upload. */
+ protected $promise;
+
+ /** @var UploadState State used to manage the upload. */
+ protected $state;
+
+ /**
+ * @param Client $client
+ * @param array $config
+ */
+ public function __construct(Client $client, array $config = [])
+ {
+ $this->client = $client;
+ $this->info = $this->loadUploadWorkflowInfo();
+ $this->config = $config + self::$defaultConfig;
+ $this->state = $this->determineState();
+ }
+
+ /**
+ * Returns the current state of the upload
+ *
+ * @return UploadState
+ */
+ public function getState()
+ {
+ return $this->state;
+ }
+
+ /**
+ * Upload the source using multipart upload operations.
+ *
+ * @return Result The result of the CompleteMultipartUpload operation.
+ * @throws \LogicException if the upload is already complete or aborted.
+ * @throws MultipartUploadException if an upload operation fails.
+ */
+ public function upload()
+ {
+ return $this->promise()->wait();
+ }
+
+ /**
+ * Upload the source asynchronously using multipart upload operations.
+ *
+ * @return PromiseInterface
+ */
+ public function promise()
+ {
+ if ($this->promise) {
+ return $this->promise;
+ }
+
+ return $this->promise = Promise\Coroutine::of(function () {
+ // Initiate the upload.
+ if ($this->state->isCompleted()) {
+ throw new \LogicException('This multipart upload has already '
+ . 'been completed or aborted.'
+ );
+ }
+
+ if (!$this->state->isInitiated()) {
+ // Execute the prepare callback.
+ if (is_callable($this->config["prepare_data_source"])) {
+ $this->config["prepare_data_source"]();
+ }
+
+ $result = (yield $this->execCommand('initiate', $this->getInitiateParams()));
+ $this->state->setUploadId(
+ $this->info['id']['upload_id'],
+ $result[$this->info['id']['upload_id']]
+ );
+ $this->state->setStatus(UploadState::INITIATED);
+ }
+
+ // Create a command pool from a generator that yields UploadPart
+ // commands for each upload part.
+ $resultHandler = $this->getResultHandler($errors);
+ $commands = new CommandPool(
+ $this->client,
+ $this->getUploadCommands($resultHandler),
+ [
+ 'concurrency' => $this->config['concurrency'],
+ 'before' => $this->config['before_upload'],
+ ]
+ );
+
+ // Execute the pool of commands concurrently, and process errors.
+ yield $commands->promise();
+ if ($errors) {
+ throw new $this->config['exception_class']($this->state, $errors);
+ }
+
+ // Complete the multipart upload.
+ yield $this->execCommand('complete', $this->getCompleteParams());
+ $this->state->setStatus(UploadState::COMPLETED);
+ })->otherwise($this->buildFailureCatch());
+ }
+
+ private function transformException($e)
+ {
+ // Throw errors from the operations as a specific Multipart error.
+ if ($e instanceof AwsException) {
+ $e = new $this->config['exception_class']($this->state, $e);
+ }
+ throw $e;
+ }
+
+ private function buildFailureCatch()
+ {
+ if (interface_exists("Throwable")) {
+ return function (\Throwable $e) {
+ return $this->transformException($e);
+ };
+ } else {
+ return function (\Exception $e) {
+ return $this->transformException($e);
+ };
+ }
+ }
+
+ protected function getConfig()
+ {
+ return $this->config;
+ }
+
+ /**
+ * Provides service-specific information about the multipart upload
+ * workflow.
+ *
+ * This array of data should include the keys: 'command', 'id', and 'part_num'.
+ *
+ * @return array
+ */
+ abstract protected function loadUploadWorkflowInfo();
+
+ /**
+ * Determines the part size to use for upload parts.
+ *
+ * Examines the provided partSize value and the source to determine the
+ * best possible part size.
+ *
+ * @throws \InvalidArgumentException if the part size is invalid.
+ *
+ * @return int
+ */
+ abstract protected function determinePartSize();
+
+ /**
+ * Uses information from the Command and Result to determine which part was
+ * uploaded and mark it as uploaded in the upload's state.
+ *
+ * @param CommandInterface $command
+ * @param ResultInterface $result
+ */
+ abstract protected function handleResult(
+ CommandInterface $command,
+ ResultInterface $result
+ );
+
+ /**
+ * Gets the service-specific parameters used to initiate the upload.
+ *
+ * @return array
+ */
+ abstract protected function getInitiateParams();
+
+ /**
+ * Gets the service-specific parameters used to complete the upload.
+ *
+ * @return array
+ */
+ abstract protected function getCompleteParams();
+
+ /**
+ * Based on the config and service-specific workflow info, creates a
+ * `Promise` for an `UploadState` object.
+ *
+ * @return PromiseInterface A `Promise` that resolves to an `UploadState`.
+ */
+ private function determineState()
+ {
+ // If the state was provided via config, then just use it.
+ if ($this->config['state'] instanceof UploadState) {
+ return $this->config['state'];
+ }
+
+ // Otherwise, construct a new state from the provided identifiers.
+ $required = $this->info['id'];
+ $id = [$required['upload_id'] => null];
+ unset($required['upload_id']);
+ foreach ($required as $key => $param) {
+ if (!$this->config[$key]) {
+ throw new IAE('You must provide a value for "' . $key . '" in '
+ . 'your config for the MultipartUploader for '
+ . $this->client->getApi()->getServiceFullName() . '.');
+ }
+ $id[$param] = $this->config[$key];
+ }
+ $state = new UploadState($id);
+ $state->setPartSize($this->determinePartSize());
+
+ return $state;
+ }
+
+ /**
+ * Executes a MUP command with all of the parameters for the operation.
+ *
+ * @param string $operation Name of the operation.
+ * @param array $params Service-specific params for the operation.
+ *
+ * @return PromiseInterface
+ */
+ protected function execCommand($operation, array $params)
+ {
+ // Create the command.
+ $command = $this->client->getCommand(
+ $this->info['command'][$operation],
+ $params + $this->state->getId()
+ );
+
+ // Execute the before callback.
+ if (is_callable($this->config["before_{$operation}"])) {
+ $this->config["before_{$operation}"]($command);
+ }
+
+ // Execute the command asynchronously and return the promise.
+ return $this->client->executeAsync($command);
+ }
+
+ /**
+ * Returns a middleware for processing responses of part upload operations.
+ *
+ * - Adds an onFulfilled callback that calls the service-specific
+ * handleResult method on the Result of the operation.
+ * - Adds an onRejected callback that adds the error to an array of errors.
+ * - Has a passedByRef $errors arg that the exceptions get added to. The
+ * caller should use that &$errors array to do error handling.
+ *
+ * @param array $errors Errors from upload operations are added to this.
+ *
+ * @return callable
+ */
+ protected function getResultHandler(&$errors = [])
+ {
+ return function (callable $handler) use (&$errors) {
+ return function (
+ CommandInterface $command,
+ RequestInterface $request = null
+ ) use ($handler, &$errors) {
+ return $handler($command, $request)->then(
+ function (ResultInterface $result) use ($command) {
+ $this->handleResult($command, $result);
+ return $result;
+ },
+ function (AwsException $e) use (&$errors) {
+ $errors[$e->getCommand()[$this->info['part_num']]] = $e;
+ return new Result();
+ }
+ );
+ };
+ };
+ }
+
+ /**
+ * Creates a generator that yields part data for the upload's source.
+ *
+ * Yields associative arrays of parameters that are ultimately merged in
+ * with others to form the complete parameters of a command. This can
+ * include the Body parameter, which is a limited stream (i.e., a Stream
+ * object, decorated with a LimitStream).
+ *
+ * @param callable $resultHandler
+ *
+ * @return \Generator
+ */
+ abstract protected function getUploadCommands(callable $resultHandler);
+}
\ No newline at end of file
diff --git a/src/Multipart/AbstractDownloader.php b/src/Multipart/AbstractDownloader.php
new file mode 100644
index 0000000000..729641e32e
--- /dev/null
+++ b/src/Multipart/AbstractDownloader.php
@@ -0,0 +1,150 @@
+source = $this->determineSource($source);
+ parent::__construct($client, $config);
+ }
+
+ /**
+ * Create a stream for a part that starts at the current position and
+ * has a length of the upload part size (or less with the final part).
+ *
+ * @param Stream $stream
+ *
+ * @return Psr7\LimitStream
+ */
+ protected function limitPartStream(Stream $stream)
+ {
+ // Limit what is read from the stream to the part size.
+ return new Psr7\LimitStream(
+ $stream,
+ $this->state->getPartSize(),
+ $this->source->tell()
+ );
+ }
+
+ protected function getUploadCommands(callable $resultHandler)
+ {
+ // Determine if the source can be seeked.
+ $seekable = $this->source->isSeekable()
+ && $this->source->getMetadata('wrapper_type') === 'plainfile';
+
+ for ($partNumber = 1; $this->isEof($seekable); $partNumber++) {
+ // If we haven't already uploaded this part, yield a new part.
+ if (!$this->state->hasPartBeenUploaded($partNumber)) {
+ $partStartPos = $this->source->tell();
+ if (!($data = $this->createPart($seekable, $partNumber))) {
+ break;
+ }
+ $command = $this->client->getCommand(
+ $this->info['command']['upload'],
+ $data + $this->state->getId()
+ );
+ $command->getHandlerList()->appendSign($resultHandler, 'mup');
+ $numberOfParts = $this->getNumberOfParts($this->state->getPartSize());
+ if (isset($numberOfParts) && $partNumber > $numberOfParts) {
+ throw new $this->config['exception_class'](
+ $this->state,
+ new AwsException(
+ "Maximum part number for this job exceeded, file has likely been corrupted." .
+ " Please restart this upload.",
+ $command
+ )
+ );
+ }
+
+ yield $command;
+ if ($this->source->tell() > $partStartPos) {
+ continue;
+ }
+ }
+
+ // Advance the source's offset if not already advanced.
+ if ($seekable) {
+ $this->source->seek(min(
+ $this->source->tell() + $this->state->getPartSize(),
+ $this->source->getSize()
+ ));
+ } else {
+ $this->source->read($this->state->getPartSize());
+ }
+ }
+ }
+
+ /**
+ * Generates the parameters for an upload part by analyzing a range of the
+ * source starting from the current offset up to the part size.
+ *
+ * @param bool $seekable
+ * @param int $number
+ *
+ * @return array|null
+ */
+ abstract protected function createPart($seekable, $number);
+
+ /**
+ * Checks if the source is at EOF.
+ *
+ * @param bool $seekable
+ *
+ * @return bool
+ */
+ private function isEof($seekable)
+ {
+ return $seekable
+ ? $this->source->tell() < $this->source->getSize()
+ : !$this->source->eof();
+ }
+
+ /**
+ * Turns the provided source into a stream and stores it.
+ *
+ * If a string is provided, it is assumed to be a filename, otherwise, it
+ * passes the value directly to `Psr7\Utils::streamFor()`.
+ *
+ * @param mixed $source
+ *
+ * @return Stream
+ */
+ private function determineSource($source)
+ {
+ // Use the contents of a file as the data source.
+ if (is_string($source)) {
+ $source = Psr7\Utils::tryFopen($source, 'r');
+ }
+
+ // Create a source stream.
+ $stream = Psr7\Utils::streamFor($source);
+ if (!$stream->isReadable()) {
+ throw new IAE('Source stream must be readable.');
+ }
+
+ return $stream;
+ }
+
+ protected function getNumberOfParts($partSize)
+ {
+ if ($sourceSize = $this->source->getSize()) {
+ return ceil($sourceSize/$partSize);
+ }
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
new file mode 100644
index 0000000000..737c3b030e
--- /dev/null
+++ b/src/S3/MultipartDownloader.php
@@ -0,0 +1,178 @@
+ null,
+ 'key' => null,
+ 'exception_class' => S3MultipartUploadException::class,
+ ]);
+ if (isset($config['track_upload']) && $config['track_upload']) {
+ $this->getState()->setProgressThresholds($this->source->getSize());
+ }
+ }
+
+ protected function loadUploadWorkflowInfo()
+ {
+ return [
+ 'command' => [
+ 'initiate' => 'CreateMultipartUpload',
+ 'upload' => 'UploadPart',
+ 'complete' => 'CompleteMultipartUpload',
+ ],
+ 'id' => [
+ 'bucket' => 'Bucket',
+ 'key' => 'Key',
+ 'upload_id' => 'UploadId',
+ ],
+ 'part_num' => 'PartNumber',
+ ];
+ }
+
+ protected function createPart($seekable, $number)
+ {
+ // Initialize the array of part data that will be returned.
+ $data = [];
+
+ // Apply custom params to UploadPart data
+ $config = $this->getConfig();
+ $params = isset($config['params']) ? $config['params'] : [];
+ foreach ($params as $k => $v) {
+ $data[$k] = $v;
+ }
+
+ $data['PartNumber'] = $number;
+
+ // Read from the source to create the body stream.
+ if ($seekable) {
+ // Case 1: Source is seekable, use lazy stream to defer work.
+ $body = $this->limitPartStream(
+ new Psr7\LazyOpenStream($this->source->getMetadata('uri'), 'r')
+ );
+ } else {
+ // Case 2: Stream is not seekable; must store in temp stream.
+ $source = $this->limitPartStream($this->source);
+ $source = $this->decorateWithHashes($source, $data);
+ $body = Psr7\Utils::streamFor();
+ Psr7\Utils::copyToStream($source, $body);
+ }
+
+ $contentLength = $body->getSize();
+
+ // Do not create a part if the body size is zero.
+ if ($contentLength === 0) {
+ return false;
+ }
+
+ $body->seek(0);
+ $data['Body'] = $body;
+
+ if (isset($config['add_content_md5'])
+ && $config['add_content_md5'] === true
+ ) {
+ $data['AddContentMD5'] = true;
+ }
+
+ $data['ContentLength'] = $contentLength;
+
+ return $data;
+ }
+
+ protected function extractETag(ResultInterface $result)
+ {
+ return $result['ETag'];
+ }
+
+ protected function getSourceMimeType()
+ {
+ if ($uri = $this->source->getMetadata('uri')) {
+ return Psr7\MimeType::fromFilename($uri)
+ ?: 'application/octet-stream';
+ }
+ }
+
+ protected function getSourceSize()
+ {
+ return $this->source->getSize();
+ }
+
+ /**
+ * Decorates a stream with a sha256 linear hashing stream.
+ *
+ * @param Stream $stream Stream to decorate.
+ * @param array $data Part data to augment with the hash result.
+ *
+ * @return Stream
+ */
+ private function decorateWithHashes(Stream $stream, array &$data)
+ {
+ // Decorate source with a hashing stream
+ $hash = new PhpHash('sha256');
+ return new HashingStream($stream, $hash, function ($result) use (&$data) {
+ $data['ContentSHA256'] = bin2hex($result);
+ });
+ }
+}
\ No newline at end of file
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
new file mode 100644
index 0000000000..902404577a
--- /dev/null
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -0,0 +1,137 @@
+ $bucket,
+ 'Key' => $key,
+ 'UploadId' => $uploadId,
+ ]);
+
+ foreach ($client->getPaginator('ListParts', $state->getId()) as $result) {
+ // Get the part size from the first part in the first result.
+ if (!$state->getPartSize()) {
+ $state->setPartSize($result->search('Parts[0].Size'));
+ }
+ // Mark all the parts returned by ListParts as uploaded.
+ foreach ($result['Parts'] as $part) {
+ $state->markPartAsUploaded($part['PartNumber'], [
+ 'PartNumber' => $part['PartNumber'],
+ 'ETag' => $part['ETag']
+ ]);
+ }
+ }
+
+ $state->setStatus(UploadState::INITIATED);
+
+ return $state;
+ }
+
+ protected function handleResult(CommandInterface $command, ResultInterface $result)
+ {
+ $this->getState()->markPartAsUploaded($command['PartNumber'], [
+ 'PartNumber' => $command['PartNumber'],
+ 'ETag' => $this->extractETag($result),
+ ]);
+
+ $this->uploadedBytes += $command["ContentLength"];
+ $this->getState()->displayProgress($this->uploadedBytes);
+ }
+
+ abstract protected function extractETag(ResultInterface $result);
+
+ protected function getCompleteParams()
+ {
+ $config = $this->getConfig();
+ $params = isset($config['params']) ? $config['params'] : [];
+
+ $params['MultipartUpload'] = [
+ 'Parts' => $this->getState()->getUploadedParts()
+ ];
+
+ return $params;
+ }
+
+ protected function determinePartSize()
+ {
+ // Make sure the part size is set.
+ $partSize = $this->getConfig()['part_size'] ?: MultipartUploader::PART_MIN_SIZE;
+
+ // Adjust the part size to be larger for known, x-large uploads.
+ if ($sourceSize = $this->getSourceSize()) {
+ $partSize = (int) max(
+ $partSize,
+ ceil($sourceSize / MultipartUploader::PART_MAX_NUM)
+ );
+ }
+
+ // Ensure that the part size follows the rules: 5 MB <= size <= 5 GB.
+ if ($partSize < MultipartUploader::PART_MIN_SIZE || $partSize > MultipartUploader::PART_MAX_SIZE) {
+ throw new \InvalidArgumentException('The part size must be no less '
+ . 'than 5 MB and no greater than 5 GB.');
+ }
+
+ return $partSize;
+ }
+
+ protected function getInitiateParams()
+ {
+ $config = $this->getConfig();
+ $params = isset($config['params']) ? $config['params'] : [];
+
+ if (isset($config['acl'])) {
+ $params['ACL'] = $config['acl'];
+ }
+
+ // Set the ContentType if not already present
+ if (empty($params['ContentType']) && $type = $this->getSourceMimeType()) {
+ $params['ContentType'] = $type;
+ }
+
+ return $params;
+ }
+
+ /**
+ * @return UploadState
+ */
+ abstract protected function getState();
+
+ /**
+ * @return array
+ */
+ abstract protected function getConfig();
+
+ /**
+ * @return int
+ */
+ abstract protected function getSourceSize();
+
+ /**
+ * @return string|null
+ */
+ abstract protected function getSourceMimeType();
+}
\ No newline at end of file
From eaccd6036798ad2f3c6ed5a8ce399d142e26bece Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Fri, 19 May 2023 14:11:32 -0400
Subject: [PATCH 22/31] ObjectDownloader + download() in S3 client trait
---
src/S3/ObjectDownloader.php | 129 +++++++++++++++++++++++++++++++++++
src/S3/S3ClientInterface.php | 12 ++++
src/S3/S3ClientTrait.php | 19 ++++++
3 files changed, 160 insertions(+)
create mode 100644 src/S3/ObjectDownloader.php
diff --git a/src/S3/ObjectDownloader.php b/src/S3/ObjectDownloader.php
new file mode 100644
index 0000000000..d91c337493
--- /dev/null
+++ b/src/S3/ObjectDownloader.php
@@ -0,0 +1,129 @@
+ null,
+ 'concurrency' => 3,
+ 'mup_threshold' => self::DEFAULT_MULTIPART_THRESHOLD,
+ 'params' => [],
+ 'part_size' => null,
+ ];
+ private $addContentMD5;
+
+ /**
+ * @param S3ClientInterface $client The S3 Client used to execute
+ * the upload command(s).
+ * @param string $bucket Bucket to upload the object, or
+ * an S3 access point ARN.
+ * @param string $key Key of the object.
+ * @param mixed $body Object data to upload. Can be a
+ * StreamInterface, PHP stream
+ * resource, or a string of data to
+ * upload.
+ * @param string $acl ACL to apply to the copy
+ * (default: private).
+ * @param array $options Options used to configure the
+ * copy process. Options passed in
+ * through 'params' are added to
+ * the sub command(s).
+ */
+ public function __construct(
+ S3ClientInterface $client,
+ $bucket,
+ $key,
+ $dest
+ ) {
+ $this->client = $client;
+ $this->bucket = $bucket;
+ $this->key = $key;
+ $this->dest = $dest;
+ }
+
+ /**
+ * @return PromiseInterface
+ */
+ public function promise()
+ {
+ // Perform a regular GetObject operation.
+ $command = $this->client->getCommand('GetObject', [
+ 'Bucket' => $this->bucket,
+ 'Key' => $this->key,
+ 'SaveAs' => $this->dest
+ ]);
+
+ return $this->client->executeAsync($command);
+ }
+
+ public function download()
+ {
+ return $this->promise()->wait();
+ }
+
+ /**
+ * Determines if the body should be uploaded using PutObject or the
+ * Multipart Upload System. It also modifies the passed-in $body as needed
+ * to support the upload.
+ *
+ * @param StreamInterface $body Stream representing the body.
+ * @param integer $threshold Minimum bytes before using Multipart.
+ *
+ * @return bool
+ */
+ private function requiresMultipart(StreamInterface &$body, $threshold)
+ {
+ // If body size known, compare to threshold to determine if Multipart.
+ if ($body->getSize() !== null) {
+ return $body->getSize() >= $threshold;
+ }
+
+ /**
+ * Handle the situation where the body size is unknown.
+ * Read up to 5MB into a buffer to determine how to upload the body.
+ * @var StreamInterface $buffer
+ */
+ $buffer = Psr7\Utils::streamFor();
+ Psr7\Utils::copyToStream($body, $buffer, MultipartUploader::PART_MIN_SIZE);
+
+ // If body < 5MB, use PutObject with the buffer.
+ if ($buffer->getSize() < MultipartUploader::PART_MIN_SIZE) {
+ $buffer->seek(0);
+ $body = $buffer;
+ return false;
+ }
+
+ // If body >= 5 MB, then use multipart. [YES]
+ if ($body->isSeekable() && $body->getMetadata('uri') !== 'php://input') {
+ // If the body is seekable, just rewind the body.
+ $body->seek(0);
+ } else {
+ // If the body is non-seekable, stitch the rewind the buffer and
+ // the partially read body together into one stream. This avoids
+ // unnecessary disc usage and does not require seeking on the
+ // original stream.
+ $buffer->seek(0);
+ $body = new Psr7\AppendStream([$buffer, $body]);
+ }
+
+ return true;
+ }
+}
+
diff --git a/src/S3/S3ClientInterface.php b/src/S3/S3ClientInterface.php
index 261d7dd3c0..f5d84dcc8c 100644
--- a/src/S3/S3ClientInterface.php
+++ b/src/S3/S3ClientInterface.php
@@ -217,6 +217,18 @@ public function uploadAsync(
array $options = []
);
+ public function download(
+ $bucket,
+ $key,
+ $dest
+ );
+
+ public function downloadAsync(
+ $bucket,
+ $key,
+ $dest
+ );
+
/**
* Copy an object of any size to a different location.
*
diff --git a/src/S3/S3ClientTrait.php b/src/S3/S3ClientTrait.php
index 5ec2f7d6a3..f404f52060 100644
--- a/src/S3/S3ClientTrait.php
+++ b/src/S3/S3ClientTrait.php
@@ -49,6 +49,25 @@ public function uploadAsync(
->promise();
}
+ public function download(
+ $bucket,
+ $key,
+ $dest
+ ) {
+ return $this
+ ->downloadAsync($bucket, $key, $dest)
+ ->wait();
+ }
+
+ public function downloadAsync(
+ $bucket,
+ $key,
+ $dest
+ ) {
+ return (new ObjectDownloader($bucket, $key, $dest))
+ ->promise();
+ }
+
/**
* @see S3ClientInterface::copy()
*/
From 3c98ca62f028ac2685c6f12b33a77b5a09cd2e99 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Wed, 24 May 2023 11:43:20 -0400
Subject: [PATCH 23/31] notes included
---
src/Multipart/AbstractDownloadManager.php | 81 ++++++------
src/Multipart/AbstractDownloader.php | 16 +--
src/Multipart/DownloadState.php | 145 ++++++++++++++++++++++
src/S3/MultipartDownloader.php | 71 ++++-------
src/S3/MultipartDownloadingTrait.php | 50 ++++----
5 files changed, 240 insertions(+), 123 deletions(-)
create mode 100644 src/Multipart/DownloadState.php
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
index 6d47ae77d6..0ca6bb2b41 100644
--- a/src/Multipart/AbstractDownloadManager.php
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -14,7 +14,7 @@
use Psr\Http\Message\RequestInterface;
/**
- * Encapsulates the execution of a multipart upload to S3 or Glacier.
+ * Encapsulates the execution of a multipart download to S3 or Glacier.
*
* @internal
*/
@@ -27,26 +27,27 @@ abstract class AbstractDownloadManager implements Promise\PromisorInterface
'part_size' => null,
'state' => null,
'concurrency' => self::DEFAULT_CONCURRENCY,
- 'prepare_data_source' => null,
+ //'prepare_data_source' => null,
'before_initiate' => null,
'before_upload' => null,
'before_complete' => null,
'exception_class' => 'Aws\Exception\MultipartUploadException',
];
+ //TO DO: check if we still need default configs
- /** @var Client Client used for the upload. */
+ /** @var Client Client used for the download. */
protected $client;
- /** @var array Configuration used to perform the upload. */
+ /** @var array Configuration used to perform the download. */
protected $config;
- /** @var array Service-specific information about the upload workflow. */
+ /** @var array Service-specific information about the download workflow. */
protected $info;
- /** @var PromiseInterface Promise that represents the multipart upload. */
+ /** @var PromiseInterface Promise that represents the multipart download. */
protected $promise;
- /** @var UploadState State used to manage the upload. */
+ /** @var DownloadState State used to manage the download. */
protected $state;
/**
@@ -56,15 +57,15 @@ abstract class AbstractDownloadManager implements Promise\PromisorInterface
public function __construct(Client $client, array $config = [])
{
$this->client = $client;
- $this->info = $this->loadUploadWorkflowInfo();
+ $this->info = $this->loadDownloadWorkflowInfo();
$this->config = $config + self::$defaultConfig;
$this->state = $this->determineState();
}
/**
- * Returns the current state of the upload
+ * Returns the current state of the download
*
- * @return UploadState
+ * @return DownloadState
*/
public function getState()
{
@@ -72,19 +73,19 @@ public function getState()
}
/**
- * Upload the source using multipart upload operations.
+ * Download the source using multipart download operations.
*
* @return Result The result of the CompleteMultipartUpload operation.
* @throws \LogicException if the upload is already complete or aborted.
* @throws MultipartUploadException if an upload operation fails.
*/
- public function upload()
+ public function download()
{
return $this->promise()->wait();
}
/**
- * Upload the source asynchronously using multipart upload operations.
+ * Download the source asynchronously using multipart download operations.
*
* @return PromiseInterface
*/
@@ -95,7 +96,7 @@ public function promise()
}
return $this->promise = Promise\Coroutine::of(function () {
- // Initiate the upload.
+ // Initiate the download.
if ($this->state->isCompleted()) {
throw new \LogicException('This multipart upload has already '
. 'been completed or aborted.'
@@ -109,22 +110,22 @@ public function promise()
}
$result = (yield $this->execCommand('initiate', $this->getInitiateParams()));
- $this->state->setUploadId(
- $this->info['id']['upload_id'],
- $result[$this->info['id']['upload_id']]
+ $this->state->setDownloadId(
+ $this->info['id']['download_id'],
+ $result[$this->info['id']['download_id']]
);
- $this->state->setStatus(UploadState::INITIATED);
+ $this->state->setStatus(DownloadState::INITIATED);
}
- // Create a command pool from a generator that yields UploadPart
- // commands for each upload part.
+ // Create a command pool from a generator that yields DownloadPart
+ // commands for each downloda part.
$resultHandler = $this->getResultHandler($errors);
$commands = new CommandPool(
$this->client,
- $this->getUploadCommands($resultHandler),
+ $this->getDownloadCommands($resultHandler),
[
'concurrency' => $this->config['concurrency'],
- 'before' => $this->config['before_upload'],
+ 'before' => $this->config['before_download'],
]
);
@@ -134,9 +135,9 @@ public function promise()
throw new $this->config['exception_class']($this->state, $errors);
}
- // Complete the multipart upload.
+ // Complete the multipart download.
yield $this->execCommand('complete', $this->getCompleteParams());
- $this->state->setStatus(UploadState::COMPLETED);
+ $this->state->setStatus(DownloadState::COMPLETED);
})->otherwise($this->buildFailureCatch());
}
@@ -168,17 +169,17 @@ protected function getConfig()
}
/**
- * Provides service-specific information about the multipart upload
+ * Provides service-specific information about the multipart download
* workflow.
*
* This array of data should include the keys: 'command', 'id', and 'part_num'.
*
* @return array
*/
- abstract protected function loadUploadWorkflowInfo();
+ abstract protected function loadDownloadWorkflowInfo();
/**
- * Determines the part size to use for upload parts.
+ * Determines the part size to use for download parts.
*
* Examines the provided partSize value and the source to determine the
* best possible part size.
@@ -191,7 +192,7 @@ abstract protected function determinePartSize();
/**
* Uses information from the Command and Result to determine which part was
- * uploaded and mark it as uploaded in the upload's state.
+ * downloaded and mark it as downloaded in the download's state.
*
* @param CommandInterface $command
* @param ResultInterface $result
@@ -202,14 +203,14 @@ abstract protected function handleResult(
);
/**
- * Gets the service-specific parameters used to initiate the upload.
+ * Gets the service-specific parameters used to initiate the download.
*
* @return array
*/
abstract protected function getInitiateParams();
/**
- * Gets the service-specific parameters used to complete the upload.
+ * Gets the service-specific parameters used to complete the download.
*
* @return array
*/
@@ -217,21 +218,21 @@ abstract protected function getCompleteParams();
/**
* Based on the config and service-specific workflow info, creates a
- * `Promise` for an `UploadState` object.
+ * `Promise` for an `DownloadState` object.
*
- * @return PromiseInterface A `Promise` that resolves to an `UploadState`.
+ * @return PromiseInterface A `Promise` that resolves to an `DownloadState`.
*/
private function determineState()
{
// If the state was provided via config, then just use it.
- if ($this->config['state'] instanceof UploadState) {
+ if ($this->config['state'] instanceof DownloadState) {
return $this->config['state'];
}
// Otherwise, construct a new state from the provided identifiers.
$required = $this->info['id'];
- $id = [$required['upload_id'] => null];
- unset($required['upload_id']);
+ $id = [$required['download_id'] => null];
+ unset($required['download_id']);
foreach ($required as $key => $param) {
if (!$this->config[$key]) {
throw new IAE('You must provide a value for "' . $key . '" in '
@@ -240,7 +241,7 @@ private function determineState()
}
$id[$param] = $this->config[$key];
}
- $state = new UploadState($id);
+ $state = new DownloadState($id);
$state->setPartSize($this->determinePartSize());
return $state;
@@ -272,7 +273,7 @@ protected function execCommand($operation, array $params)
}
/**
- * Returns a middleware for processing responses of part upload operations.
+ * Returns a middleware for processing responses of part download operations.
*
* - Adds an onFulfilled callback that calls the service-specific
* handleResult method on the Result of the operation.
@@ -280,7 +281,7 @@ protected function execCommand($operation, array $params)
* - Has a passedByRef $errors arg that the exceptions get added to. The
* caller should use that &$errors array to do error handling.
*
- * @param array $errors Errors from upload operations are added to this.
+ * @param array $errors Errors from download operations are added to this.
*
* @return callable
*/
@@ -306,7 +307,7 @@ function (AwsException $e) use (&$errors) {
}
/**
- * Creates a generator that yields part data for the upload's source.
+ * Creates a generator that yields part data for the download's source.
*
* Yields associative arrays of parameters that are ultimately merged in
* with others to form the complete parameters of a command. This can
@@ -317,5 +318,5 @@ function (AwsException $e) use (&$errors) {
*
* @return \Generator
*/
- abstract protected function getUploadCommands(callable $resultHandler);
+ abstract protected function getDownloadCommands(callable $resultHandler);
}
\ No newline at end of file
diff --git a/src/Multipart/AbstractDownloader.php b/src/Multipart/AbstractDownloader.php
index 729641e32e..b30e46d39e 100644
--- a/src/Multipart/AbstractDownloader.php
+++ b/src/Multipart/AbstractDownloader.php
@@ -7,9 +7,9 @@
use InvalidArgumentException as IAE;
use Psr\Http\Message\StreamInterface as Stream;
-abstract class AbstractDownloader extends AbstractUploadManager
+abstract class AbstractDownloader extends AbstractDownloadManager
{
- /** @var Stream Source of the data to be uploaded. */
+ /** @var Stream Source of the data to be downloaded. */
protected $source;
/**
@@ -25,7 +25,7 @@ public function __construct(Client $client, $source, array $config = [])
/**
* Create a stream for a part that starts at the current position and
- * has a length of the upload part size (or less with the final part).
+ * has a length of the download part size (or less with the final part).
*
* @param Stream $stream
*
@@ -41,21 +41,21 @@ protected function limitPartStream(Stream $stream)
);
}
- protected function getUploadCommands(callable $resultHandler)
+ protected function getDownloadCommands(callable $resultHandler)
{
// Determine if the source can be seeked.
$seekable = $this->source->isSeekable()
&& $this->source->getMetadata('wrapper_type') === 'plainfile';
for ($partNumber = 1; $this->isEof($seekable); $partNumber++) {
- // If we haven't already uploaded this part, yield a new part.
- if (!$this->state->hasPartBeenUploaded($partNumber)) {
+ // If we haven't already downloaded this part, yield a new part.
+ if (!$this->state->hasPartBeenDownloaded($partNumber)) {
$partStartPos = $this->source->tell();
if (!($data = $this->createPart($seekable, $partNumber))) {
break;
}
$command = $this->client->getCommand(
- $this->info['command']['upload'],
+ $this->info['command']['download'],
$data + $this->state->getId()
);
$command->getHandlerList()->appendSign($resultHandler, 'mup');
@@ -90,7 +90,7 @@ protected function getUploadCommands(callable $resultHandler)
}
/**
- * Generates the parameters for an upload part by analyzing a range of the
+ * Generates the parameters for an download part by analyzing a range of the
* source starting from the current offset up to the part size.
*
* @param bool $seekable
diff --git a/src/Multipart/DownloadState.php b/src/Multipart/DownloadState.php
new file mode 100644
index 0000000000..567adbe2d2
--- /dev/null
+++ b/src/Multipart/DownloadState.php
@@ -0,0 +1,145 @@
+id = $id;
+ }
+
+ /**
+ * Get the download's ID, which is a tuple of parameters that can uniquely
+ * identify the download.
+ *
+ * @return array
+ */
+ public function getId()
+ {
+ return $this->id;
+ }
+
+ /**
+ * Set's the "download_id", or 3rd part of the download's ID. This typically
+ * only needs to be done after initiating an download.
+ *
+ * @param string $key The param key of the download_id.
+ * @param string $value The param value of the download_id.
+ */
+ public function setDownloadId($key, $value)
+ {
+ $this->id[$key] = $value;
+ }
+
+ /**
+ * Get the part size.
+ *
+ * @return int
+ */
+ public function getPartSize()
+ {
+ return $this->partSize;
+ }
+
+ /**
+ * Set the part size.
+ *
+ * @param $partSize int Size of download parts.
+ */
+ public function setPartSize($partSize)
+ {
+ $this->partSize = $partSize;
+ }
+
+ /**
+ * Marks a part as being downloaded.
+ *
+ * @param int $partNumber The part number.
+ * @param array $partData Data from the download operation that needs to be
+ * recalled during the complete operation.
+ */
+ public function markPartAsDownloaded($partNumber, array $partData = [])
+ {
+ $this->DownloadedParts[$partNumber] = $partData;
+ }
+
+ /**
+ * Returns whether a part has been downloaded.
+ *
+ * @param int $partNumber The part number.
+ *
+ * @return bool
+ */
+ public function hasPartBeenDownloaded($partNumber)
+ {
+ return isset($this->downloadedParts[$partNumber]);
+ }
+
+ /**
+ * Returns a sorted list of all the downloaded parts.
+ *
+ * @return array
+ */
+ public function getDownloadedParts()
+ {
+ ksort($this->downloadedParts);
+ return $this->downloadedParts;
+ }
+
+ /**
+ * Set the status of the download.
+ *
+ * @param int $status Status is an integer code defined by the constants
+ * CREATED, INITIATED, and COMPLETED on this class.
+ */
+
+ public function setStatus($status)
+ {
+ $this->status = $status;
+ }
+
+ /**
+ * Determines whether the download state is in the INITIATED status.
+ *
+ * @return bool
+ */
+ public function isInitiated()
+ {
+ return $this->status === self::INITIATED;
+ }
+
+ /**
+ * Determines whether the download state is in the COMPLETED status.
+ *
+ * @return bool
+ */
+ public function isCompleted()
+ {
+ return $this->status === self::COMPLETED;
+ }
+}
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
index 737c3b030e..6187ee0727 100644
--- a/src/S3/MultipartDownloader.php
+++ b/src/S3/MultipartDownloader.php
@@ -1,8 +1,9 @@
getObjectInfo($client, $config['bucket'], $config['key']);
+ echo $result['ContentLength'];
parent::__construct($client, $source, array_change_key_case($config) + [
'bucket' => null,
'key' => null,
@@ -75,18 +38,26 @@ public function __construct(
}
}
- protected function loadUploadWorkflowInfo()
+ public function getObjectInfo($client, $bucket, $key)
+ {
+ return $client->headObject([
+ 'Bucket' => $bucket,
+ 'Key' => $key,
+ ]);
+ }
+
+ protected function loadDownloadWorkflowInfo()
{
return [
'command' => [
- 'initiate' => 'CreateMultipartUpload',
- 'upload' => 'UploadPart',
- 'complete' => 'CompleteMultipartUpload',
+ 'initiate' => 'getObject',
+ 'download' => 'getObject',
+ 'complete' => 'getObject',
],
'id' => [
'bucket' => 'Bucket',
'key' => 'Key',
- 'upload_id' => 'UploadId',
+ 'download_id' => 'DownloadId',
],
'part_num' => 'PartNumber',
];
@@ -97,7 +68,7 @@ protected function createPart($seekable, $number)
// Initialize the array of part data that will be returned.
$data = [];
- // Apply custom params to UploadPart data
+ // Apply custom params to DownloadPart data
$config = $this->getConfig();
$params = isset($config['params']) ? $config['params'] : [];
foreach ($params as $k => $v) {
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
index 902404577a..388eeb7db8 100644
--- a/src/S3/MultipartDownloadingTrait.php
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -2,34 +2,34 @@
namespace Aws\S3;
use Aws\CommandInterface;
-use Aws\Multipart\UploadState;
+use Aws\Multipart\DownloadState;
use Aws\ResultInterface;
trait MultipartDownloadingTrait
{
- private $uploadedBytes = 0;
+ private $downloadedBytes = 0;
/**
- * Creates an UploadState object for a multipart upload by querying the
- * service for the specified upload's information.
+ * Creates an DownloadState object for a multipart download by querying the
+ * service for the specified download's information.
*
- * @param S3ClientInterface $client S3Client used for the upload.
- * @param string $bucket Bucket for the multipart upload.
- * @param string $key Object key for the multipart upload.
- * @param string $uploadId Upload ID for the multipart upload.
+ * @param S3ClientInterface $client S3Client used for the download.
+ * @param string $bucket Bucket for the multipart download.
+ * @param string $key Object key for the multipart download.
+ * @param string $downloadId Download ID for the multipart download.
*
- * @return UploadState
+ * @return DownloadState
*/
public static function getStateFromService(
S3ClientInterface $client,
$bucket,
$key,
- $uploadId
+ $downloadId
) {
- $state = new UploadState([
+ $state = new DownloadState([
'Bucket' => $bucket,
'Key' => $key,
- 'UploadId' => $uploadId,
+ 'DownloadId' => $downloadId,
]);
foreach ($client->getPaginator('ListParts', $state->getId()) as $result) {
@@ -37,29 +37,29 @@ public static function getStateFromService(
if (!$state->getPartSize()) {
$state->setPartSize($result->search('Parts[0].Size'));
}
- // Mark all the parts returned by ListParts as uploaded.
+ // Mark all the parts returned by ListParts as downloaded.
foreach ($result['Parts'] as $part) {
- $state->markPartAsUploaded($part['PartNumber'], [
+ $state->markPartAsDownloaded($part['PartNumber'], [
'PartNumber' => $part['PartNumber'],
'ETag' => $part['ETag']
]);
}
}
- $state->setStatus(UploadState::INITIATED);
+ $state->setStatus(DownloadState::INITIATED);
return $state;
}
protected function handleResult(CommandInterface $command, ResultInterface $result)
{
- $this->getState()->markPartAsUploaded($command['PartNumber'], [
+ $this->getState()->markPartAsDownloaded($command['PartNumber'], [
'PartNumber' => $command['PartNumber'],
'ETag' => $this->extractETag($result),
]);
- $this->uploadedBytes += $command["ContentLength"];
- $this->getState()->displayProgress($this->uploadedBytes);
+ $this->downloadedBytes += $command["ContentLength"];
+ $this->getState()->displayProgress($this->downloadedBytes);
}
abstract protected function extractETag(ResultInterface $result);
@@ -69,8 +69,8 @@ protected function getCompleteParams()
$config = $this->getConfig();
$params = isset($config['params']) ? $config['params'] : [];
- $params['MultipartUpload'] = [
- 'Parts' => $this->getState()->getUploadedParts()
+ $params['MultipartDownload'] = [
+ 'Parts' => $this->getState()->getDownloadedParts()
];
return $params;
@@ -79,18 +79,18 @@ protected function getCompleteParams()
protected function determinePartSize()
{
// Make sure the part size is set.
- $partSize = $this->getConfig()['part_size'] ?: MultipartUploader::PART_MIN_SIZE;
+ $partSize = $this->getConfig()['part_size'] ?: MultipartDownloader::PART_MIN_SIZE;
- // Adjust the part size to be larger for known, x-large uploads.
+ // Adjust the part size to be larger for known, x-large downloads.
if ($sourceSize = $this->getSourceSize()) {
$partSize = (int) max(
$partSize,
- ceil($sourceSize / MultipartUploader::PART_MAX_NUM)
+ ceil($sourceSize / MultipartDownloader::PART_MAX_NUM)
);
}
// Ensure that the part size follows the rules: 5 MB <= size <= 5 GB.
- if ($partSize < MultipartUploader::PART_MIN_SIZE || $partSize > MultipartUploader::PART_MAX_SIZE) {
+ if ($partSize < MultipartDownloader::PART_MIN_SIZE || $partSize > MultipartDownloader::PART_MAX_SIZE) {
throw new \InvalidArgumentException('The part size must be no less '
. 'than 5 MB and no greater than 5 GB.');
}
@@ -116,7 +116,7 @@ protected function getInitiateParams()
}
/**
- * @return UploadState
+ * @return DownloadState
*/
abstract protected function getState();
From b71fc6ca288e06c4802bec0187f0d25619083bc3 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 5 Jun 2023 00:04:36 -0400
Subject: [PATCH 24/31] works with parts (includes notes/incorrectly named
methods)
---
src/Exception/MultipartDownloadException.php | 64 +++++++++
src/Multipart/AbstractDownloadManager.php | 84 ++++++------
src/Multipart/AbstractDownloader.php | 77 +++++------
src/Multipart/DownloadState.php | 110 +++++++++++-----
.../S3MultipartDownloadException.php | 85 ++++++++++++
src/S3/MultipartDownloader.php | 124 +++++++++++-------
src/S3/MultipartDownloadingTrait.php | 80 +++++++----
tests/S3/MultipartDownloaderTest.php | 20 +++
8 files changed, 449 insertions(+), 195 deletions(-)
create mode 100644 src/Exception/MultipartDownloadException.php
create mode 100644 src/S3/Exception/S3MultipartDownloadException.php
create mode 100644 tests/S3/MultipartDownloaderTest.php
diff --git a/src/Exception/MultipartDownloadException.php b/src/Exception/MultipartDownloadException.php
new file mode 100644
index 0000000000..ad35714f37
--- /dev/null
+++ b/src/Exception/MultipartDownloadException.php
@@ -0,0 +1,64 @@
+ 'uploading parts to']);
+ $msg .= ". The following parts had errors:\n";
+ /** @var $error AwsException */
+ foreach ($prev as $part => $error) {
+ $msg .= "- Part {$part}: " . $error->getMessage(). "\n";
+ }
+ } elseif ($prev instanceof AwsException) {
+ switch ($prev->getCommand()->getName()) {
+ case 'CreateMultipartUpload':
+ case 'InitiateMultipartUpload':
+ $action = 'initiating';
+ break;
+ case 'CompleteMultipartUpload':
+ $action = 'completing';
+ break;
+ }
+ if (isset($action)) {
+ $msg = strtr($msg, ['performing' => $action]);
+ }
+ $msg .= ": {$prev->getMessage()}";
+ }
+
+ if (!$prev instanceof \Exception) {
+ $prev = null;
+ }
+
+ parent::__construct($msg, 0, $prev);
+ $this->state = $state;
+ }
+
+ /**
+ * Get the state of the transfer
+ *
+ * @return DownloadState
+ */
+ public function getState()
+ {
+ return $this->state;
+ }
+}
+
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
index 0ca6bb2b41..991947c034 100644
--- a/src/Multipart/AbstractDownloadManager.php
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -5,7 +5,7 @@
use Aws\CommandInterface;
use Aws\CommandPool;
use Aws\Exception\AwsException;
-use Aws\Exception\MultipartUploadException;
+use Aws\Exception\MultipartDownloadException;
use Aws\Result;
use Aws\ResultInterface;
use GuzzleHttp\Promise;
@@ -14,7 +14,7 @@
use Psr\Http\Message\RequestInterface;
/**
- * Encapsulates the execution of a multipart download to S3 or Glacier.
+ * Encapsulates the execution of a multipart upload to S3 or Glacier.
*
* @internal
*/
@@ -27,27 +27,26 @@ abstract class AbstractDownloadManager implements Promise\PromisorInterface
'part_size' => null,
'state' => null,
'concurrency' => self::DEFAULT_CONCURRENCY,
- //'prepare_data_source' => null,
+ 'prepare_data_source' => null,
'before_initiate' => null,
'before_upload' => null,
'before_complete' => null,
- 'exception_class' => 'Aws\Exception\MultipartUploadException',
+ 'exception_class' => 'Aws\Exception\MultipartDownloadException',
];
- //TO DO: check if we still need default configs
- /** @var Client Client used for the download. */
+ /** @var Client Client used for the upload. */
protected $client;
- /** @var array Configuration used to perform the download. */
+ /** @var array Configuration used to perform the upload. */
protected $config;
- /** @var array Service-specific information about the download workflow. */
+ /** @var array Service-specific information about the upload workflow. */
protected $info;
- /** @var PromiseInterface Promise that represents the multipart download. */
+ /** @var PromiseInterface Promise that represents the multipart upload. */
protected $promise;
- /** @var DownloadState State used to manage the download. */
+ /** @var UploadState State used to manage the upload. */
protected $state;
/**
@@ -57,15 +56,15 @@ abstract class AbstractDownloadManager implements Promise\PromisorInterface
public function __construct(Client $client, array $config = [])
{
$this->client = $client;
- $this->info = $this->loadDownloadWorkflowInfo();
+ $this->info = $this->loadUploadWorkflowInfo();
$this->config = $config + self::$defaultConfig;
$this->state = $this->determineState();
}
/**
- * Returns the current state of the download
+ * Returns the current state of the upload
*
- * @return DownloadState
+ * @return UploadState
*/
public function getState()
{
@@ -73,7 +72,7 @@ public function getState()
}
/**
- * Download the source using multipart download operations.
+ * Upload the source using multipart upload operations.
*
* @return Result The result of the CompleteMultipartUpload operation.
* @throws \LogicException if the upload is already complete or aborted.
@@ -85,7 +84,7 @@ public function download()
}
/**
- * Download the source asynchronously using multipart download operations.
+ * Upload the source asynchronously using multipart upload operations.
*
* @return PromiseInterface
*/
@@ -96,7 +95,7 @@ public function promise()
}
return $this->promise = Promise\Coroutine::of(function () {
- // Initiate the download.
+ // Initiate the upload.
if ($this->state->isCompleted()) {
throw new \LogicException('This multipart upload has already '
. 'been completed or aborted.'
@@ -110,22 +109,25 @@ public function promise()
}
$result = (yield $this->execCommand('initiate', $this->getInitiateParams()));
- $this->state->setDownloadId(
- $this->info['id']['download_id'],
- $result[$this->info['id']['download_id']]
+ $this->determineSourceSize($result['ContentLength']);
+ $this->setStreamPosArray($result['ContentLength']);
+ $this->state->setUploadId(
+ $this->info['id']['upload_id'],
+ $result[$this->info['id']['upload_id']]
);
+// print_r($this->info);
$this->state->setStatus(DownloadState::INITIATED);
}
- // Create a command pool from a generator that yields DownloadPart
- // commands for each downloda part.
+ // Create a command pool from a generator that yields UploadPart
+ // commands for each upload part.
$resultHandler = $this->getResultHandler($errors);
$commands = new CommandPool(
$this->client,
- $this->getDownloadCommands($resultHandler),
+ $this->getUploadCommands($resultHandler),
[
'concurrency' => $this->config['concurrency'],
- 'before' => $this->config['before_download'],
+ 'before' => $this->config['before_upload'],
]
);
@@ -135,8 +137,8 @@ public function promise()
throw new $this->config['exception_class']($this->state, $errors);
}
- // Complete the multipart download.
- yield $this->execCommand('complete', $this->getCompleteParams());
+ // Complete the multipart upload.
+// yield $this->execCommand('complete', $this->getCompleteParams());
$this->state->setStatus(DownloadState::COMPLETED);
})->otherwise($this->buildFailureCatch());
}
@@ -169,17 +171,19 @@ protected function getConfig()
}
/**
- * Provides service-specific information about the multipart download
+ * Provides service-specific information about the multipart upload
* workflow.
*
* This array of data should include the keys: 'command', 'id', and 'part_num'.
*
* @return array
*/
- abstract protected function loadDownloadWorkflowInfo();
+ abstract protected function loadUploadWorkflowInfo();
+
+ abstract protected function determineSourceSize($size);
/**
- * Determines the part size to use for download parts.
+ * Determines the part size to use for upload parts.
*
* Examines the provided partSize value and the source to determine the
* best possible part size.
@@ -192,7 +196,7 @@ abstract protected function determinePartSize();
/**
* Uses information from the Command and Result to determine which part was
- * downloaded and mark it as downloaded in the download's state.
+ * uploaded and mark it as uploaded in the upload's state.
*
* @param CommandInterface $command
* @param ResultInterface $result
@@ -203,14 +207,14 @@ abstract protected function handleResult(
);
/**
- * Gets the service-specific parameters used to initiate the download.
+ * Gets the service-specific parameters used to initiate the upload.
*
* @return array
*/
abstract protected function getInitiateParams();
/**
- * Gets the service-specific parameters used to complete the download.
+ * Gets the service-specific parameters used to complete the upload.
*
* @return array
*/
@@ -218,9 +222,9 @@ abstract protected function getCompleteParams();
/**
* Based on the config and service-specific workflow info, creates a
- * `Promise` for an `DownloadState` object.
+ * `Promise` for an `UploadState` object.
*
- * @return PromiseInterface A `Promise` that resolves to an `DownloadState`.
+ * @return PromiseInterface A `Promise` that resolves to an `UploadState`.
*/
private function determineState()
{
@@ -231,8 +235,8 @@ private function determineState()
// Otherwise, construct a new state from the provided identifiers.
$required = $this->info['id'];
- $id = [$required['download_id'] => null];
- unset($required['download_id']);
+ $id = [$required['upload_id'] => null];
+ unset($required['upload_id']);
foreach ($required as $key => $param) {
if (!$this->config[$key]) {
throw new IAE('You must provide a value for "' . $key . '" in '
@@ -273,7 +277,7 @@ protected function execCommand($operation, array $params)
}
/**
- * Returns a middleware for processing responses of part download operations.
+ * Returns a middleware for processing responses of part upload operations.
*
* - Adds an onFulfilled callback that calls the service-specific
* handleResult method on the Result of the operation.
@@ -281,7 +285,7 @@ protected function execCommand($operation, array $params)
* - Has a passedByRef $errors arg that the exceptions get added to. The
* caller should use that &$errors array to do error handling.
*
- * @param array $errors Errors from download operations are added to this.
+ * @param array $errors Errors from upload operations are added to this.
*
* @return callable
*/
@@ -307,7 +311,7 @@ function (AwsException $e) use (&$errors) {
}
/**
- * Creates a generator that yields part data for the download's source.
+ * Creates a generator that yields part data for the upload's source.
*
* Yields associative arrays of parameters that are ultimately merged in
* with others to form the complete parameters of a command. This can
@@ -318,5 +322,5 @@ function (AwsException $e) use (&$errors) {
*
* @return \Generator
*/
- abstract protected function getDownloadCommands(callable $resultHandler);
-}
\ No newline at end of file
+ abstract protected function getUploadCommands(callable $resultHandler);
+}
diff --git a/src/Multipart/AbstractDownloader.php b/src/Multipart/AbstractDownloader.php
index b30e46d39e..f356a4f333 100644
--- a/src/Multipart/AbstractDownloader.php
+++ b/src/Multipart/AbstractDownloader.php
@@ -9,9 +9,11 @@
abstract class AbstractDownloader extends AbstractDownloadManager
{
- /** @var Stream Source of the data to be downloaded. */
+ /** @var Stream Source of the data to be uploaded. */
protected $source;
+ protected $position = 0;
+
/**
* @param Client $client
* @param mixed $source
@@ -19,13 +21,13 @@ abstract class AbstractDownloader extends AbstractDownloadManager
*/
public function __construct(Client $client, $source, array $config = [])
{
- $this->source = $this->determineSource($source);
+// $this->source = $this->determineSource($source);
parent::__construct($client, $config);
}
/**
* Create a stream for a part that starts at the current position and
- * has a length of the download part size (or less with the final part).
+ * has a length of the upload part size (or less with the final part).
*
* @param Stream $stream
*
@@ -41,21 +43,18 @@ protected function limitPartStream(Stream $stream)
);
}
- protected function getDownloadCommands(callable $resultHandler)
+ protected function getUploadCommands(callable $resultHandler)
{
// Determine if the source can be seeked.
- $seekable = $this->source->isSeekable()
- && $this->source->getMetadata('wrapper_type') === 'plainfile';
-
- for ($partNumber = 1; $this->isEof($seekable); $partNumber++) {
- // If we haven't already downloaded this part, yield a new part.
- if (!$this->state->hasPartBeenDownloaded($partNumber)) {
- $partStartPos = $this->source->tell();
- if (!($data = $this->createPart($seekable, $partNumber))) {
+ for ($partNumber = 1; $this->isEof($this->position); $partNumber++) {
+ // If we haven't already uploaded this part, yield a new part.
+ if (!$this->state->hasPartBeenUploaded($partNumber)) {
+ $partStartPos = $this->position;
+ if (!($data = $this->createPart($partStartPos, $partNumber))) {
break;
}
$command = $this->client->getCommand(
- $this->info['command']['download'],
+ $this->info['command']['upload'],
$data + $this->state->getId()
);
$command->getHandlerList()->appendSign($resultHandler, 'mup');
@@ -72,25 +71,18 @@ protected function getDownloadCommands(callable $resultHandler)
}
yield $command;
- if ($this->source->tell() > $partStartPos) {
- continue;
- }
+// if ($this->source->tell() > $partStartPos) {
+// continue;
+// }
}
// Advance the source's offset if not already advanced.
- if ($seekable) {
- $this->source->seek(min(
- $this->source->tell() + $this->state->getPartSize(),
- $this->source->getSize()
- ));
- } else {
- $this->source->read($this->state->getPartSize());
- }
+ $this->position += $this->state->getPartSize();
}
}
/**
- * Generates the parameters for an download part by analyzing a range of the
+ * Generates the parameters for an upload part by analyzing a range of the
* source starting from the current offset up to the part size.
*
* @param bool $seekable
@@ -107,11 +99,9 @@ abstract protected function createPart($seekable, $number);
*
* @return bool
*/
- private function isEof($seekable)
+ private function isEof($position)
{
- return $seekable
- ? $this->source->tell() < $this->source->getSize()
- : !$this->source->eof();
+ return $position <= $this->sourceSize;
}
/**
@@ -124,26 +114,25 @@ private function isEof($seekable)
*
* @return Stream
*/
- private function determineSource($source)
+ protected function determineSourceSize($size)
{
- // Use the contents of a file as the data source.
- if (is_string($source)) {
- $source = Psr7\Utils::tryFopen($source, 'r');
- }
-
- // Create a source stream.
- $stream = Psr7\Utils::streamFor($source);
- if (!$stream->isReadable()) {
- throw new IAE('Source stream must be readable.');
- }
-
- return $stream;
+ echo 'abstract downloader size: ' . $size;
+ $this->sourceSize = $size;
+// $generator = function ($bytes) {
+// for ($i = 0; $i < $bytes; $i++) {
+// yield '.';
+// }
+// };
+//
+// $iter = $generator($this->sourceSize);
+// $stream = Psr7\Utils::streamFor($iter);
+// $this->source = $stream;
}
protected function getNumberOfParts($partSize)
{
- if ($sourceSize = $this->source->getSize()) {
- return ceil($sourceSize/$partSize);
+ if ($this->sourceSize) {
+ return ceil($this->sourceSize/$partSize);
}
return null;
}
diff --git a/src/Multipart/DownloadState.php b/src/Multipart/DownloadState.php
index 567adbe2d2..8153b01972 100644
--- a/src/Multipart/DownloadState.php
+++ b/src/Multipart/DownloadState.php
@@ -2,10 +2,10 @@
namespace Aws\Multipart;
/**
- * Representation of the multipart download.
+ * Representation of the multipart upload.
*
- * This object keeps track of the state of the download, including the status and
- * which parts have been downloaded.
+ * This object keeps track of the state of the upload, including the status and
+ * which parts have been uploaded.
*/
class DownloadState
{
@@ -13,20 +13,36 @@ class DownloadState
const INITIATED = 1;
const COMPLETED = 2;
- /** @var array Params used to identity the download. */
+ protected $progressBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"
+ ];
+
+ /** @var array Params used to identity the upload. */
private $id;
- /** @var int Part size being used by the download. */
+ /** @var int Part size being used by the upload. */
private $partSize;
- /** @var array Parts that have been downloaded. */
- private $downloadedParts = [];
+ /** @var array Parts that have been uploaded. */
+ private $uploadedParts = [];
- /** @var int Identifies the status the download. */
+ /** @var int Identifies the status the upload. */
private $status = self::CREATED;
+ private $progressThresholds = [];
+
+// private $displayUploadProgress;
+
/**
- * @param array $id Params used to identity the download.
+ * @param array $id Params used to identity the upload.
*/
public function __construct(array $id)
{
@@ -34,8 +50,8 @@ public function __construct(array $id)
}
/**
- * Get the download's ID, which is a tuple of parameters that can uniquely
- * identify the download.
+ * Get the upload's ID, which is a tuple of parameters that can uniquely
+ * identify the upload.
*
* @return array
*/
@@ -45,14 +61,15 @@ public function getId()
}
/**
- * Set's the "download_id", or 3rd part of the download's ID. This typically
- * only needs to be done after initiating an download.
+ * Set's the "upload_id", or 3rd part of the upload's ID. This typically
+ * only needs to be done after initiating an upload.
*
- * @param string $key The param key of the download_id.
- * @param string $value The param value of the download_id.
+ * @param string $key The param key of the upload_id.
+ * @param string $value The param value of the upload_id.
*/
- public function setDownloadId($key, $value)
+ public function setUploadId($key, $value)
{
+ // i don't think i need this, instead i need to be sending the size to here?
$this->id[$key] = $value;
}
@@ -69,50 +86,79 @@ public function getPartSize()
/**
* Set the part size.
*
- * @param $partSize int Size of download parts.
+ * @param $partSize int Size of upload parts.
*/
public function setPartSize($partSize)
{
$this->partSize = $partSize;
}
+ public function setProgressThresholds($totalSize)
+ {
+ if(!is_int($totalSize)) {
+ throw new \InvalidArgumentException('The total size of the upload must be an int.');
+ }
+
+ $this->progressThresholds[0] = 0;
+ for ($i=1;$i<=8;$i++) {
+ $this->progressThresholds []= round($totalSize*($i/8));
+ }
+ $this->progressBar = array_combine($this->progressThresholds, $this->progressBar);
+ return $this->progressThresholds;
+ }
+
+ public function displayProgress($totalUploaded)
+ {
+ if(!is_int($totalUploaded)) {
+ throw new \InvalidArgumentException('The size of the bytes being uploaded must be an int.');
+ }
+
+ while ($this->progressThresholds
+ && !empty($this->progressBar)
+ && $totalUploaded >= array_key_first($this->progressBar))
+ {
+ echo $this->progressBar[array_key_first($this->progressBar)];
+ unset($this->progressBar[array_key_first($this->progressBar)]);
+ }
+ }
+
/**
- * Marks a part as being downloaded.
+ * Marks a part as being uploaded.
*
* @param int $partNumber The part number.
- * @param array $partData Data from the download operation that needs to be
+ * @param array $partData Data from the upload operation that needs to be
* recalled during the complete operation.
*/
- public function markPartAsDownloaded($partNumber, array $partData = [])
+ public function markPartAsUploaded($partNumber, array $partData = [])
{
- $this->DownloadedParts[$partNumber] = $partData;
+ $this->uploadedParts[$partNumber] = $partData;
}
/**
- * Returns whether a part has been downloaded.
+ * Returns whether a part has been uploaded.
*
* @param int $partNumber The part number.
*
* @return bool
*/
- public function hasPartBeenDownloaded($partNumber)
+ public function hasPartBeenUploaded($partNumber)
{
- return isset($this->downloadedParts[$partNumber]);
+ return isset($this->uploadedParts[$partNumber]);
}
/**
- * Returns a sorted list of all the downloaded parts.
+ * Returns a sorted list of all the uploaded parts.
*
* @return array
*/
- public function getDownloadedParts()
+ public function getUploadedParts()
{
- ksort($this->downloadedParts);
- return $this->downloadedParts;
+ ksort($this->uploadedParts);
+ return $this->uploadedParts;
}
/**
- * Set the status of the download.
+ * Set the status of the upload.
*
* @param int $status Status is an integer code defined by the constants
* CREATED, INITIATED, and COMPLETED on this class.
@@ -124,7 +170,7 @@ public function setStatus($status)
}
/**
- * Determines whether the download state is in the INITIATED status.
+ * Determines whether the upload state is in the INITIATED status.
*
* @return bool
*/
@@ -134,7 +180,7 @@ public function isInitiated()
}
/**
- * Determines whether the download state is in the COMPLETED status.
+ * Determines whether the upload state is in the COMPLETED status.
*
* @return bool
*/
@@ -142,4 +188,4 @@ public function isCompleted()
{
return $this->status === self::COMPLETED;
}
-}
+}
\ No newline at end of file
diff --git a/src/S3/Exception/S3MultipartDownloadException.php b/src/S3/Exception/S3MultipartDownloadException.php
new file mode 100644
index 0000000000..410a747a90
--- /dev/null
+++ b/src/S3/Exception/S3MultipartDownloadException.php
@@ -0,0 +1,85 @@
+collectPathInfo($error->getCommand());
+ } elseif ($prev instanceof AwsException) {
+ $this->collectPathInfo($prev->getCommand());
+ }
+ parent::__construct($state, $prev);
+ }
+
+ /**
+ * Get the Bucket information of the transfer object
+ *
+ * @return string|null Returns null when 'Bucket' information
+ * is unavailable.
+ */
+ public function getBucket()
+ {
+ return $this->bucket;
+ }
+
+ /**
+ * Get the Key information of the transfer object
+ *
+ * @return string|null Returns null when 'Key' information
+ * is unavailable.
+ */
+ public function getKey()
+ {
+ return $this->key;
+ }
+
+ /**
+ * Get the source file name of the transfer object
+ *
+ * @return string|null Returns null when metadata of the stream
+ * wrapped in 'Body' parameter is unavailable.
+ */
+ public function getSourceFileName()
+ {
+ return $this->filename;
+ }
+
+ /**
+ * Collect file path information when accessible. (Bucket, Key)
+ *
+ * @param CommandInterface $cmd
+ */
+ private function collectPathInfo(CommandInterface $cmd)
+ {
+ if (empty($this->bucket) && isset($cmd['Bucket'])) {
+ $this->bucket = $cmd['Bucket'];
+ }
+ if (empty($this->key) && isset($cmd['Key'])) {
+ $this->key = $cmd['Key'];
+ }
+ if (empty($this->filename) && isset($cmd['Body'])) {
+ $this->filename = $cmd['Body']->getMetadata('uri');
+ }
+ }
+}
+
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
index 6187ee0727..1a7f1fb36e 100644
--- a/src/S3/MultipartDownloader.php
+++ b/src/S3/MultipartDownloader.php
@@ -1,17 +1,16 @@
getObjectInfo($client, $config['bucket'], $config['key']);
- echo $result['ContentLength'];
+ $this->destStream = $this->createDestStream($source);
parent::__construct($client, $source, array_change_key_case($config) + [
'bucket' => null,
'key' => null,
- 'exception_class' => S3MultipartUploadException::class,
+ 'exception_class' => S3MultipartDownloadException::class,
]);
- if (isset($config['track_upload']) && $config['track_upload']) {
- $this->getState()->setProgressThresholds($this->source->getSize());
- }
+// if (isset($config['track_upload']) && $config['track_upload']) {
+// $this->getState()->setProgressThresholds($this->source->getSize());
+// }
}
- public function getObjectInfo($client, $bucket, $key)
- {
- return $client->headObject([
- 'Bucket' => $bucket,
- 'Key' => $key,
- ]);
- }
-
- protected function loadDownloadWorkflowInfo()
+ protected function loadUploadWorkflowInfo()
{
return [
'command' => [
- 'initiate' => 'getObject',
- 'download' => 'getObject',
- 'complete' => 'getObject',
+ 'initiate' => 'HeadObject',
+ 'upload' => 'GetObject',
+ 'complete' => 'CompleteMultipartUpload',
],
'id' => [
'bucket' => 'Bucket',
'key' => 'Key',
- 'download_id' => 'DownloadId',
+ 'upload_id' => 'UploadId',
],
'part_num' => 'PartNumber',
];
}
- protected function createPart($seekable, $number)
+ protected function createPart($partStartPos, $number)
{
// Initialize the array of part data that will be returned.
$data = [];
- // Apply custom params to DownloadPart data
+// echo 'create part position: ' . $partStartPos;
+
+ // Apply custom params to UploadPart data
$config = $this->getConfig();
$params = isset($config['params']) ? $config['params'] : [];
foreach ($params as $k => $v) {
@@ -77,38 +109,12 @@ protected function createPart($seekable, $number)
$data['PartNumber'] = $number;
- // Read from the source to create the body stream.
- if ($seekable) {
- // Case 1: Source is seekable, use lazy stream to defer work.
- $body = $this->limitPartStream(
- new Psr7\LazyOpenStream($this->source->getMetadata('uri'), 'r')
- );
- } else {
- // Case 2: Stream is not seekable; must store in temp stream.
- $source = $this->limitPartStream($this->source);
- $source = $this->decorateWithHashes($source, $data);
- $body = Psr7\Utils::streamFor();
- Psr7\Utils::copyToStream($source, $body);
- }
-
- $contentLength = $body->getSize();
-
- // Do not create a part if the body size is zero.
- if ($contentLength === 0) {
- return false;
- }
-
- $body->seek(0);
- $data['Body'] = $body;
-
if (isset($config['add_content_md5'])
&& $config['add_content_md5'] === true
) {
$data['AddContentMD5'] = true;
}
- $data['ContentLength'] = $contentLength;
-
return $data;
}
@@ -130,6 +136,17 @@ protected function getSourceSize()
return $this->source->getSize();
}
+ public function setStreamPosArray($sourceSize)
+ {
+ $parts = ceil($sourceSize/$this->state->getPartSize());
+ $position = 0;
+ for ($i=1;$i<=$parts;$i++) {
+ $this->StreamPosArray [$i]= $position;
+ $position += $this->state->getPartSize();
+ }
+ print_r($this->StreamPosArray);
+ }
+
/**
* Decorates a stream with a sha256 linear hashing stream.
*
@@ -146,4 +163,9 @@ private function decorateWithHashes(Stream $stream, array &$data)
$data['ContentSHA256'] = bin2hex($result);
});
}
-}
\ No newline at end of file
+
+ protected function createDestStream($filePath)
+ {
+ return new Psr7\LazyOpenStream($filePath, 'w');
+ }
+}
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
index 388eeb7db8..84ac31ed73 100644
--- a/src/S3/MultipartDownloadingTrait.php
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -4,19 +4,20 @@
use Aws\CommandInterface;
use Aws\Multipart\DownloadState;
use Aws\ResultInterface;
+use GuzzleHttp\Psr7;
trait MultipartDownloadingTrait
{
- private $downloadedBytes = 0;
+ private $uploadedBytes = 0;
/**
- * Creates an DownloadState object for a multipart download by querying the
- * service for the specified download's information.
+ * Creates an UploadState object for a multipart upload by querying the
+ * service for the specified upload's information.
*
- * @param S3ClientInterface $client S3Client used for the download.
- * @param string $bucket Bucket for the multipart download.
- * @param string $key Object key for the multipart download.
- * @param string $downloadId Download ID for the multipart download.
+ * @param S3ClientInterface $client S3Client used for the upload.
+ * @param string $bucket Bucket for the multipart upload.
+ * @param string $key Object key for the multipart upload.
+ * @param string $uploadId Upload ID for the multipart upload.
*
* @return DownloadState
*/
@@ -24,12 +25,12 @@ public static function getStateFromService(
S3ClientInterface $client,
$bucket,
$key,
- $downloadId
+ $uploadId
) {
$state = new DownloadState([
'Bucket' => $bucket,
'Key' => $key,
- 'DownloadId' => $downloadId,
+ 'UploadId' => $uploadId,
]);
foreach ($client->getPaginator('ListParts', $state->getId()) as $result) {
@@ -37,9 +38,9 @@ public static function getStateFromService(
if (!$state->getPartSize()) {
$state->setPartSize($result->search('Parts[0].Size'));
}
- // Mark all the parts returned by ListParts as downloaded.
+ // Mark all the parts returned by ListParts as uploaded.
foreach ($result['Parts'] as $part) {
- $state->markPartAsDownloaded($part['PartNumber'], [
+ $state->markPartAsUploaded($part['PartNumber'], [
'PartNumber' => $part['PartNumber'],
'ETag' => $part['ETag']
]);
@@ -53,13 +54,25 @@ public static function getStateFromService(
protected function handleResult(CommandInterface $command, ResultInterface $result)
{
- $this->getState()->markPartAsDownloaded($command['PartNumber'], [
+ $this->getState()->markPartAsUploaded($command['PartNumber'], [
'PartNumber' => $command['PartNumber'],
'ETag' => $this->extractETag($result),
]);
+// $bodyStream = Psr7\Utils::streamFor($result['Body']);
+//
+// $this->destStream->write($bodyStream->read(5242880));
+// $this->destStream->seek(MultipartDownloader::PART_MIN_SIZE);
+//
+// $this->uploadedBytes += $command["ContentLength"];
+ $this->writeDestStream($command['PartNumber'], $result['Body']);
+ $this->getState()->displayProgress($this->uploadedBytes);
+ }
- $this->downloadedBytes += $command["ContentLength"];
- $this->getState()->displayProgress($this->downloadedBytes);
+ protected function writeDestStream($partNum, $body)
+ {
+ $bodyStream = Psr7\Utils::streamFor($body);
+ $this->destStream->seek($this->StreamPosArray[$partNum]);
+ $this->destStream->write($bodyStream->read(5242880));
}
abstract protected function extractETag(ResultInterface $result);
@@ -69,8 +82,8 @@ protected function getCompleteParams()
$config = $this->getConfig();
$params = isset($config['params']) ? $config['params'] : [];
- $params['MultipartDownload'] = [
- 'Parts' => $this->getState()->getDownloadedParts()
+ $params['MultipartUpload'] = [
+ 'Parts' => $this->getState()->getUploadedParts()
];
return $params;
@@ -81,13 +94,13 @@ protected function determinePartSize()
// Make sure the part size is set.
$partSize = $this->getConfig()['part_size'] ?: MultipartDownloader::PART_MIN_SIZE;
- // Adjust the part size to be larger for known, x-large downloads.
- if ($sourceSize = $this->getSourceSize()) {
- $partSize = (int) max(
- $partSize,
- ceil($sourceSize / MultipartDownloader::PART_MAX_NUM)
- );
- }
+ // Adjust the part size to be larger for known, x-large uploads.
+// if ($sourceSize = $this->getSourceSize()) {
+// $partSize = (int) max(
+// $partSize,
+// ceil($sourceSize / MultipartDownloader::PART_MAX_NUM)
+// );
+// }
// Ensure that the part size follows the rules: 5 MB <= size <= 5 GB.
if ($partSize < MultipartDownloader::PART_MIN_SIZE || $partSize > MultipartDownloader::PART_MAX_SIZE) {
@@ -108,15 +121,26 @@ protected function getInitiateParams()
}
// Set the ContentType if not already present
- if (empty($params['ContentType']) && $type = $this->getSourceMimeType()) {
- $params['ContentType'] = $type;
- }
+// if (empty($params['ContentType']) && $type = $this->getSourceMimeType()) {
+// $params['ContentType'] = $type;
+// }
return $params;
}
+ public function setStreamPosArray($sourceSize)
+ {
+ $parts = ceil($sourceSize/$this->partSize);
+ $position = 0;
+ for ($i=1;$i<=$parts;$i++) {
+ $this->StreamPosArray []= $position;
+ $position += $this->partSize;
+ }
+ print_r($this->StreamPosArray);
+ }
+
/**
- * @return DownloadState
+ * @return UploadState
*/
abstract protected function getState();
@@ -134,4 +158,4 @@ abstract protected function getSourceSize();
* @return string|null
*/
abstract protected function getSourceMimeType();
-}
\ No newline at end of file
+}
diff --git a/tests/S3/MultipartDownloaderTest.php b/tests/S3/MultipartDownloaderTest.php
new file mode 100644
index 0000000000..c22e072110
--- /dev/null
+++ b/tests/S3/MultipartDownloaderTest.php
@@ -0,0 +1,20 @@
+
Date: Mon, 5 Jun 2023 11:35:40 -0400
Subject: [PATCH 25/31] starting to use multipartDownloadType config option
(comments/misnamed methods)
Works with $config['multipartdownloadtype'] === 'Part' for now since abstract downloader and download state identify parts with their part #. I also need to work on adding to the stream based on ranges/not parts.
---
src/Multipart/AbstractDownloadManager.php | 2 +-
src/Multipart/AbstractDownloader.php | 18 ------
src/Multipart/DownloadState.php | 45 -------------
src/S3/MultipartDownloader.php | 79 ++++++++++-------------
src/S3/MultipartDownloadingTrait.php | 42 ++++--------
5 files changed, 47 insertions(+), 139 deletions(-)
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
index 991947c034..f7a1f8a697 100644
--- a/src/Multipart/AbstractDownloadManager.php
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -110,7 +110,7 @@ public function promise()
$result = (yield $this->execCommand('initiate', $this->getInitiateParams()));
$this->determineSourceSize($result['ContentLength']);
- $this->setStreamPosArray($result['ContentLength']);
+ $this->setStreamPositionArray($result['ContentLength']);
$this->state->setUploadId(
$this->info['id']['upload_id'],
$result[$this->info['id']['upload_id']]
diff --git a/src/Multipart/AbstractDownloader.php b/src/Multipart/AbstractDownloader.php
index f356a4f333..efd36076a1 100644
--- a/src/Multipart/AbstractDownloader.php
+++ b/src/Multipart/AbstractDownloader.php
@@ -25,24 +25,6 @@ public function __construct(Client $client, $source, array $config = [])
parent::__construct($client, $config);
}
- /**
- * Create a stream for a part that starts at the current position and
- * has a length of the upload part size (or less with the final part).
- *
- * @param Stream $stream
- *
- * @return Psr7\LimitStream
- */
- protected function limitPartStream(Stream $stream)
- {
- // Limit what is read from the stream to the part size.
- return new Psr7\LimitStream(
- $stream,
- $this->state->getPartSize(),
- $this->source->tell()
- );
- }
-
protected function getUploadCommands(callable $resultHandler)
{
// Determine if the source can be seeked.
diff --git a/src/Multipart/DownloadState.php b/src/Multipart/DownloadState.php
index 8153b01972..8452cc1016 100644
--- a/src/Multipart/DownloadState.php
+++ b/src/Multipart/DownloadState.php
@@ -13,18 +13,6 @@ class DownloadState
const INITIATED = 1;
const COMPLETED = 2;
- protected $progressBar = [
- "Transfer initiated...\n| | 0.0%\n",
- "|== | 12.5%\n",
- "|===== | 25.0%\n",
- "|======= | 37.5%\n",
- "|========== | 50.0%\n",
- "|============ | 62.5%\n",
- "|=============== | 75.0%\n",
- "|================= | 87.5%\n",
- "|====================| 100.0%\nTransfer complete!\n"
- ];
-
/** @var array Params used to identity the upload. */
private $id;
@@ -37,10 +25,6 @@ class DownloadState
/** @var int Identifies the status the upload. */
private $status = self::CREATED;
- private $progressThresholds = [];
-
-// private $displayUploadProgress;
-
/**
* @param array $id Params used to identity the upload.
*/
@@ -93,35 +77,6 @@ public function setPartSize($partSize)
$this->partSize = $partSize;
}
- public function setProgressThresholds($totalSize)
- {
- if(!is_int($totalSize)) {
- throw new \InvalidArgumentException('The total size of the upload must be an int.');
- }
-
- $this->progressThresholds[0] = 0;
- for ($i=1;$i<=8;$i++) {
- $this->progressThresholds []= round($totalSize*($i/8));
- }
- $this->progressBar = array_combine($this->progressThresholds, $this->progressBar);
- return $this->progressThresholds;
- }
-
- public function displayProgress($totalUploaded)
- {
- if(!is_int($totalUploaded)) {
- throw new \InvalidArgumentException('The size of the bytes being uploaded must be an int.');
- }
-
- while ($this->progressThresholds
- && !empty($this->progressBar)
- && $totalUploaded >= array_key_first($this->progressBar))
- {
- echo $this->progressBar[array_key_first($this->progressBar)];
- unset($this->progressBar[array_key_first($this->progressBar)]);
- }
- }
-
/**
* Marks a part as being uploaded.
*
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
index 1a7f1fb36e..e9e93a18fc 100644
--- a/src/S3/MultipartDownloader.php
+++ b/src/S3/MultipartDownloader.php
@@ -10,7 +10,7 @@
use Aws\S3\Exception\S3MultipartDownloadException;
/**
- * Encapsulates the execution of a multipart upload to S3 or Glacier.
+ * Encapsulates the execution of a multipart download to S3.
*/
class MultipartDownloader extends AbstractDownloader
{
@@ -21,11 +21,11 @@ class MultipartDownloader extends AbstractDownloader
const PART_MAX_NUM = 10000;
/**
- * Creates a multipart upload for an S3 object.
+ * Creates a multipart download for an S3 object.
*
* The valid configuration options are as follows:
*
- * - acl: (string) ACL to set on the object being upload. Objects are
+ * - acl: (string) ACL to set on the object being download. Objects are
* private by default.
* - before_complete: (callable) Callback to invoke before the
* `CompleteMultipartUpload` operation. The callback should have a
@@ -57,23 +57,31 @@ class MultipartDownloader extends AbstractDownloader
* options are ignored.
*
* @param S3ClientInterface $client Client used for the upload.
- * @param mixed $source Source of the data to upload.
+ * @param mixed $dest Destination for data to download.
* @param array $config Configuration used to perform the upload.
*/
public function __construct(
S3ClientInterface $client,
- $source,
+ $dest,
array $config = []
) {
- $this->destStream = $this->createDestStream($source);
- parent::__construct($client, $source, array_change_key_case($config) + [
+ $this->destStream = $this->createDestStream($dest);
+// if (isset($config['multipartDownloadType'])) {
+// if ($config['multipartDownloadType'] == 'Part') {
+// if (isset($config['Range'])) {
+// echo 'do something!!!';
+// }
+// } elseif ($config['multipartDownloadType'] == 'Range') {
+// if (isset($config['Range'])) {
+// echo 'do something!!!';
+// }
+// }
+// }
+ parent::__construct($client, $dest, array_change_key_case($config) + [
'bucket' => null,
'key' => null,
'exception_class' => S3MultipartDownloadException::class,
]);
-// if (isset($config['track_upload']) && $config['track_upload']) {
-// $this->getState()->setProgressThresholds($this->source->getSize());
-// }
}
protected function loadUploadWorkflowInfo()
@@ -82,7 +90,7 @@ protected function loadUploadWorkflowInfo()
'command' => [
'initiate' => 'HeadObject',
'upload' => 'GetObject',
- 'complete' => 'CompleteMultipartUpload',
+// 'complete' => 'CompleteMultipartUpload',
],
'id' => [
'bucket' => 'Bucket',
@@ -107,7 +115,18 @@ protected function createPart($partStartPos, $number)
$data[$k] = $v;
}
- $data['PartNumber'] = $number;
+ if (isset($config['multipartdownloadtype'])
+ && $config['multipartdownloadtype'] === 'Part') {
+ $data['PartNumber'] = $number;
+ echo 'parts';
+ } elseif (isset($config['multipartdownloadtype'])
+ && $config['multipartdownloadtype'] === 'Range') {
+ $partEndPos = $partStartPos+self::PART_MIN_SIZE;
+ $data['Range'] = 'bytes='.$partStartPos.'-'.$partEndPos;
+ echo 'ranges';
+ }
+
+// $data['PartNumber'] = $number;
if (isset($config['add_content_md5'])
&& $config['add_content_md5'] === true
@@ -123,45 +142,15 @@ protected function extractETag(ResultInterface $result)
return $result['ETag'];
}
- protected function getSourceMimeType()
- {
- if ($uri = $this->source->getMetadata('uri')) {
- return Psr7\MimeType::fromFilename($uri)
- ?: 'application/octet-stream';
- }
- }
-
- protected function getSourceSize()
- {
- return $this->source->getSize();
- }
-
- public function setStreamPosArray($sourceSize)
+ public function setStreamPositionArray($sourceSize)
{
$parts = ceil($sourceSize/$this->state->getPartSize());
$position = 0;
for ($i=1;$i<=$parts;$i++) {
- $this->StreamPosArray [$i]= $position;
+ $this->streamPositionArray [$i]= $position;
$position += $this->state->getPartSize();
}
- print_r($this->StreamPosArray);
- }
-
- /**
- * Decorates a stream with a sha256 linear hashing stream.
- *
- * @param Stream $stream Stream to decorate.
- * @param array $data Part data to augment with the hash result.
- *
- * @return Stream
- */
- private function decorateWithHashes(Stream $stream, array &$data)
- {
- // Decorate source with a hashing stream
- $hash = new PhpHash('sha256');
- return new HashingStream($stream, $hash, function ($result) use (&$data) {
- $data['ContentSHA256'] = bin2hex($result);
- });
+ print_r($this->streamPositionArray);
}
protected function createDestStream($filePath)
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
index 84ac31ed73..6331287176 100644
--- a/src/S3/MultipartDownloadingTrait.php
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -8,8 +8,6 @@
trait MultipartDownloadingTrait
{
- private $uploadedBytes = 0;
-
/**
* Creates an UploadState object for a multipart upload by querying the
* service for the specified upload's information.
@@ -58,20 +56,14 @@ protected function handleResult(CommandInterface $command, ResultInterface $resu
'PartNumber' => $command['PartNumber'],
'ETag' => $this->extractETag($result),
]);
-// $bodyStream = Psr7\Utils::streamFor($result['Body']);
-//
-// $this->destStream->write($bodyStream->read(5242880));
-// $this->destStream->seek(MultipartDownloader::PART_MIN_SIZE);
-//
-// $this->uploadedBytes += $command["ContentLength"];
+
$this->writeDestStream($command['PartNumber'], $result['Body']);
- $this->getState()->displayProgress($this->uploadedBytes);
}
protected function writeDestStream($partNum, $body)
{
$bodyStream = Psr7\Utils::streamFor($body);
- $this->destStream->seek($this->StreamPosArray[$partNum]);
+ $this->destStream->seek($this->streamPositionArray[$partNum]);
$this->destStream->write($bodyStream->read(5242880));
}
@@ -128,16 +120,16 @@ protected function getInitiateParams()
return $params;
}
- public function setStreamPosArray($sourceSize)
- {
- $parts = ceil($sourceSize/$this->partSize);
- $position = 0;
- for ($i=1;$i<=$parts;$i++) {
- $this->StreamPosArray []= $position;
- $position += $this->partSize;
- }
- print_r($this->StreamPosArray);
- }
+// public function setStreamPosArray($sourceSize)
+// {
+// $parts = ceil($sourceSize/$this->partSize);
+// $position = 0;
+// for ($i=1;$i<=$parts;$i++) {
+// $this->StreamPosArray []= $position;
+// $position += $this->partSize;
+// }
+// print_r($this->streamPositionArray);
+// }
/**
* @return UploadState
@@ -148,14 +140,4 @@ abstract protected function getState();
* @return array
*/
abstract protected function getConfig();
-
- /**
- * @return int
- */
- abstract protected function getSourceSize();
-
- /**
- * @return string|null
- */
- abstract protected function getSourceMimeType();
}
From 87bc27d2721dd4899cac44256991f2cbf3d369ef Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Sat, 10 Jun 2023 00:11:57 -0400
Subject: [PATCH 26/31] starting parts/ranges
---
src/Multipart/AbstractDownloadManager.php | 7 ++
src/Multipart/AbstractDownloader.php | 78 ++++++++++++++---------
src/S3/MultipartDownloader.php | 12 ++--
src/S3/MultipartDownloadingTrait.php | 21 ++++--
4 files changed, 80 insertions(+), 38 deletions(-)
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
index f7a1f8a697..26a865de7c 100644
--- a/src/Multipart/AbstractDownloadManager.php
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -109,6 +109,9 @@ public function promise()
}
$result = (yield $this->execCommand('initiate', $this->getInitiateParams()));
+// echo $result;
+// echo $result['ContentRange'];
+ // if range or part config, end it here.
$this->determineSourceSize($result['ContentLength']);
$this->setStreamPositionArray($result['ContentLength']);
$this->state->setUploadId(
@@ -117,6 +120,10 @@ public function promise()
);
// print_r($this->info);
$this->state->setStatus(DownloadState::INITIATED);
+// $this->getState()->markPartAsUploaded($command['PartNumber'], [
+// 'PartNumber' => $command['PartNumber'],
+// 'ETag' => $this->extractETag($result),
+// ]);
}
// Create a command pool from a generator that yields UploadPart
diff --git a/src/Multipart/AbstractDownloader.php b/src/Multipart/AbstractDownloader.php
index efd36076a1..14a9856844 100644
--- a/src/Multipart/AbstractDownloader.php
+++ b/src/Multipart/AbstractDownloader.php
@@ -22,44 +22,62 @@ abstract class AbstractDownloader extends AbstractDownloadManager
public function __construct(Client $client, $source, array $config = [])
{
// $this->source = $this->determineSource($source);
+ $this->config = $config;
parent::__construct($client, $config);
}
protected function getUploadCommands(callable $resultHandler)
{
- // Determine if the source can be seeked.
- for ($partNumber = 1; $this->isEof($this->position); $partNumber++) {
- // If we haven't already uploaded this part, yield a new part.
- if (!$this->state->hasPartBeenUploaded($partNumber)) {
- $partStartPos = $this->position;
- if (!($data = $this->createPart($partStartPos, $partNumber))) {
- break;
- }
- $command = $this->client->getCommand(
- $this->info['command']['upload'],
- $data + $this->state->getId()
- );
- $command->getHandlerList()->appendSign($resultHandler, 'mup');
- $numberOfParts = $this->getNumberOfParts($this->state->getPartSize());
- if (isset($numberOfParts) && $partNumber > $numberOfParts) {
- throw new $this->config['exception_class'](
- $this->state,
- new AwsException(
- "Maximum part number for this job exceeded, file has likely been corrupted." .
- " Please restart this upload.",
- $command
- )
+ // partnumber - single object calls
+ if (isset($this->config['partnumber'])) {
+ $data = $this->createPart('partNumber', $this->config['partnumber']);
+ $command = $this->client->getCommand(
+ $this->info['command']['upload'],
+ $data + $this->state->getId()
+ );
+ $command->getHandlerList()->appendSign($resultHandler, 'mup');
+ yield $command;
+ } elseif (isset($this->config['range'])){ // range - single object calls
+ $data = $this->createPart($this->position, $this->config['partnumber']);
+ $command = $this->client->getCommand(
+ $this->info['command']['upload'],
+ $data + $this->state->getId()
+
+ );
+ $command->getHandlerList()->appendSign($resultHandler, 'mup');
+ yield $command;
+ } else { // multipart calls
+ // Determine if the source can be seeked.
+ for ($partNumber = 1; $this->isEof($this->position); $partNumber++) {
+ // If we haven't already uploaded this part, yield a new part.
+ if (!$this->state->hasPartBeenUploaded($partNumber)) {
+ $partStartPos = $this->position;
+ if (!($data = $this->createPart($partStartPos, $partNumber))) {
+ break;
+ }
+ $command = $this->client->getCommand(
+ $this->info['command']['upload'],
+ $data + $this->state->getId()
);
+ $command->getHandlerList()->appendSign($resultHandler, 'mup');
+ $numberOfParts = $this->getNumberOfParts($this->state->getPartSize());
+ if (isset($numberOfParts) && $partNumber > $numberOfParts) {
+ throw new $this->config['exception_class'](
+ $this->state,
+ new AwsException(
+ "Maximum part number for this job exceeded, file has likely been corrupted." .
+ " Please restart this upload.",
+ $command
+ )
+ );
+ }
+
+ yield $command;
}
- yield $command;
-// if ($this->source->tell() > $partStartPos) {
-// continue;
-// }
+ // Advance the source's offset if not already advanced.
+ $this->position += $this->state->getPartSize();
}
-
- // Advance the source's offset if not already advanced.
- $this->position += $this->state->getPartSize();
}
}
@@ -98,7 +116,7 @@ private function isEof($position)
*/
protected function determineSourceSize($size)
{
- echo 'abstract downloader size: ' . $size;
+// echo 'abstract downloader size: ' . $size;
$this->sourceSize = $size;
// $generator = function ($bytes) {
// for ($i = 0; $i < $bytes; $i++) {
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
index e9e93a18fc..89ae46ada5 100644
--- a/src/S3/MultipartDownloader.php
+++ b/src/S3/MultipartDownloader.php
@@ -88,7 +88,7 @@ protected function loadUploadWorkflowInfo()
{
return [
'command' => [
- 'initiate' => 'HeadObject',
+ 'initiate' => 'GetObject',
'upload' => 'GetObject',
// 'complete' => 'CompleteMultipartUpload',
],
@@ -101,7 +101,7 @@ protected function loadUploadWorkflowInfo()
];
}
- protected function createPart($partStartPos, $number)
+ protected function createPart($type, $number)
{
// Initialize the array of part data that will be returned.
$data = [];
@@ -126,7 +126,11 @@ protected function createPart($partStartPos, $number)
echo 'ranges';
}
-// $data['PartNumber'] = $number;
+// if (isset($config['partNumber'])) {
+// $data['PartNumber'] = $config['partNumber'];
+// }
+
+ $data['PartNumber'] = $number;
if (isset($config['add_content_md5'])
&& $config['add_content_md5'] === true
@@ -150,7 +154,7 @@ public function setStreamPositionArray($sourceSize)
$this->streamPositionArray [$i]= $position;
$position += $this->state->getPartSize();
}
- print_r($this->streamPositionArray);
+// print_r($this->streamPositionArray);
}
protected function createDestStream($filePath)
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
index 6331287176..e87e77e462 100644
--- a/src/S3/MultipartDownloadingTrait.php
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -112,10 +112,23 @@ protected function getInitiateParams()
$params['ACL'] = $config['acl'];
}
- // Set the ContentType if not already present
-// if (empty($params['ContentType']) && $type = $this->getSourceMimeType()) {
-// $params['ContentType'] = $type;
-// }
+ if (isset($config['partnumber'])) {
+ $params['PartNumber'] = $config['partnumber'];
+ echo 'PartNumber';
+ } elseif (isset($config['range'])) {
+ $params['Range'] = $config['range'];
+ echo 'Range';
+ } elseif (isset($config['multipartdownloadtype']) && $config['multipartdownloadtype'] == 'Range') {
+ $params['Range'] = 'bytes=0-'.MultipartDownloader::PART_MIN_SIZE;
+ echo 'multipartdownloadtype';
+ } else {
+ $params['PartNumber'] = 1;
+ echo 'part num 1';
+ }
+
+// $params['PartNumber'] = $config['partnumber'];
+
+// $params['Range'] = 'bytes=0-1225';
return $params;
}
From 88e46036692037a64108fea31966bd1cf1747a17 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Tue, 13 Jun 2023 11:52:41 -0400
Subject: [PATCH 27/31] all config options work (needs refactoring)
---
src/Multipart/AbstractDownloadManager.php | 68 ++++++++++---------
src/Multipart/AbstractDownloader.php | 44 +++----------
src/Multipart/DownloadState.php | 2 +-
src/S3/MultipartDownloader.php | 59 ++++++++---------
src/S3/MultipartDownloadingTrait.php | 80 ++++++++++++++++-------
5 files changed, 128 insertions(+), 125 deletions(-)
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
index 26a865de7c..56b821f634 100644
--- a/src/Multipart/AbstractDownloadManager.php
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -107,47 +107,49 @@ public function promise()
if (is_callable($this->config["prepare_data_source"])) {
$this->config["prepare_data_source"]();
}
-
- $result = (yield $this->execCommand('initiate', $this->getInitiateParams()));
-// echo $result;
-// echo $result['ContentRange'];
+ $type = $this->getUploadType();
+ $result = (yield $this->execCommand('initiate', $this->getInitiateParams($type)));
// if range or part config, end it here.
- $this->determineSourceSize($result['ContentLength']);
- $this->setStreamPositionArray($result['ContentLength']);
+ $this->determineSourceSize($result['ContentRange']);
$this->state->setUploadId(
$this->info['id']['upload_id'],
$result[$this->info['id']['upload_id']]
);
-// print_r($this->info);
$this->state->setStatus(DownloadState::INITIATED);
-// $this->getState()->markPartAsUploaded($command['PartNumber'], [
-// 'PartNumber' => $command['PartNumber'],
-// 'ETag' => $this->extractETag($result),
-// ]);
+ if (isset($type['type'])){
+ $this->handleResult(1, $result);
+ } else {
+ $this->handleResult($type['configParam'], $result);
+ }
}
- // Create a command pool from a generator that yields UploadPart
- // commands for each upload part.
- $resultHandler = $this->getResultHandler($errors);
- $commands = new CommandPool(
- $this->client,
- $this->getUploadCommands($resultHandler),
- [
- 'concurrency' => $this->config['concurrency'],
- 'before' => $this->config['before_upload'],
- ]
- );
-
- // Execute the pool of commands concurrently, and process errors.
- yield $commands->promise();
- if ($errors) {
- throw new $this->config['exception_class']($this->state, $errors);
- }
+ if (isset($this->config['partnumber'])
+ or isset($this->config['range'])
+ or $result['PartsCount']==1){
+ $this->state->setStatus(DownloadState::COMPLETED);
+ } else {
+ // Create a command pool from a generator that yields UploadPart
+ // commands for each upload part.
+ $resultHandler = $this->getResultHandler($errors);
+ $commands = new CommandPool(
+ $this->client,
+ $this->getUploadCommands($resultHandler),
+ [
+ 'concurrency' => $this->config['concurrency'],
+ 'before' => $this->config['before_upload'],
+ ]
+ );
- // Complete the multipart upload.
+ // Execute the pool of commands concurrently, and process errors.
+ yield $commands->promise();
+ if ($errors) {
+ throw new $this->config['exception_class']($this->state, $errors);
+ }
+
+ // Complete the multipart upload.
// yield $this->execCommand('complete', $this->getCompleteParams());
- $this->state->setStatus(DownloadState::COMPLETED);
- })->otherwise($this->buildFailureCatch());
+ $this->state->setStatus(DownloadState::COMPLETED);
+ }})->otherwise($this->buildFailureCatch());
}
private function transformException($e)
@@ -218,7 +220,9 @@ abstract protected function handleResult(
*
* @return array
*/
- abstract protected function getInitiateParams();
+ abstract protected function getInitiateParams($type);
+
+ abstract protected function getUploadType();
/**
* Gets the service-specific parameters used to complete the upload.
diff --git a/src/Multipart/AbstractDownloader.php b/src/Multipart/AbstractDownloader.php
index 14a9856844..03eb98e83c 100644
--- a/src/Multipart/AbstractDownloader.php
+++ b/src/Multipart/AbstractDownloader.php
@@ -28,29 +28,10 @@ public function __construct(Client $client, $source, array $config = [])
protected function getUploadCommands(callable $resultHandler)
{
- // partnumber - single object calls
- if (isset($this->config['partnumber'])) {
- $data = $this->createPart('partNumber', $this->config['partnumber']);
- $command = $this->client->getCommand(
- $this->info['command']['upload'],
- $data + $this->state->getId()
- );
- $command->getHandlerList()->appendSign($resultHandler, 'mup');
- yield $command;
- } elseif (isset($this->config['range'])){ // range - single object calls
- $data = $this->createPart($this->position, $this->config['partnumber']);
- $command = $this->client->getCommand(
- $this->info['command']['upload'],
- $data + $this->state->getId()
-
- );
- $command->getHandlerList()->appendSign($resultHandler, 'mup');
- yield $command;
- } else { // multipart calls
- // Determine if the source can be seeked.
- for ($partNumber = 1; $this->isEof($this->position); $partNumber++) {
- // If we haven't already uploaded this part, yield a new part.
- if (!$this->state->hasPartBeenUploaded($partNumber)) {
+ // Determine if the source can be seeked.
+ for ($partNumber = 1; $this->isEof($this->position); $partNumber++) {
+ // If we haven't already uploaded this part, yield a new part.
+ if (!$this->state->hasPartBeenUploaded($partNumber)) {
$partStartPos = $this->position;
if (!($data = $this->createPart($partStartPos, $partNumber))) {
break;
@@ -60,7 +41,7 @@ protected function getUploadCommands(callable $resultHandler)
$data + $this->state->getId()
);
$command->getHandlerList()->appendSign($resultHandler, 'mup');
- $numberOfParts = $this->getNumberOfParts($this->state->getPartSize());
+ $numberOfParts = ($this->getNumberOfParts($this->state->getPartSize()));
if (isset($numberOfParts) && $partNumber > $numberOfParts) {
throw new $this->config['exception_class'](
$this->state,
@@ -78,7 +59,6 @@ protected function getUploadCommands(callable $resultHandler)
// Advance the source's offset if not already advanced.
$this->position += $this->state->getPartSize();
}
- }
}
/**
@@ -114,19 +94,11 @@ private function isEof($position)
*
* @return Stream
*/
- protected function determineSourceSize($size)
+ protected function determineSourceSize($range)
{
-// echo 'abstract downloader size: ' . $size;
+ $size = substr($range, strpos($range, "/") + 1);
$this->sourceSize = $size;
-// $generator = function ($bytes) {
-// for ($i = 0; $i < $bytes; $i++) {
-// yield '.';
-// }
-// };
-//
-// $iter = $generator($this->sourceSize);
-// $stream = Psr7\Utils::streamFor($iter);
-// $this->source = $stream;
+ $this->setStreamPositionArray($this->sourceSize);
}
protected function getNumberOfParts($partSize)
diff --git a/src/Multipart/DownloadState.php b/src/Multipart/DownloadState.php
index 8452cc1016..6fa28016b7 100644
--- a/src/Multipart/DownloadState.php
+++ b/src/Multipart/DownloadState.php
@@ -80,7 +80,7 @@ public function setPartSize($partSize)
/**
* Marks a part as being uploaded.
*
- * @param int $partNumber The part number.
+ * @param string $partNumber The part number.
* @param array $partData Data from the upload operation that needs to be
* recalled during the complete operation.
*/
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
index 89ae46ada5..5416c31130 100644
--- a/src/S3/MultipartDownloader.php
+++ b/src/S3/MultipartDownloader.php
@@ -66,17 +66,6 @@ public function __construct(
array $config = []
) {
$this->destStream = $this->createDestStream($dest);
-// if (isset($config['multipartDownloadType'])) {
-// if ($config['multipartDownloadType'] == 'Part') {
-// if (isset($config['Range'])) {
-// echo 'do something!!!';
-// }
-// } elseif ($config['multipartDownloadType'] == 'Range') {
-// if (isset($config['Range'])) {
-// echo 'do something!!!';
-// }
-// }
-// }
parent::__construct($client, $dest, array_change_key_case($config) + [
'bucket' => null,
'key' => null,
@@ -101,13 +90,11 @@ protected function loadUploadWorkflowInfo()
];
}
- protected function createPart($type, $number)
+ protected function createPart($partStartPos, $number)
{
// Initialize the array of part data that will be returned.
$data = [];
-// echo 'create part position: ' . $partStartPos;
-
// Apply custom params to UploadPart data
$config = $this->getConfig();
$params = isset($config['params']) ? $config['params'] : [];
@@ -115,23 +102,21 @@ protected function createPart($type, $number)
$data[$k] = $v;
}
- if (isset($config['multipartdownloadtype'])
- && $config['multipartdownloadtype'] === 'Part') {
- $data['PartNumber'] = $number;
- echo 'parts';
- } elseif (isset($config['multipartdownloadtype'])
- && $config['multipartdownloadtype'] === 'Range') {
+// } elseif (isset($config['multipartdownloadtype'])
+// && $config['multipartdownloadtype'] === 'Range') {
+// $partEndPos = $partStartPos+self::PART_MIN_SIZE;
+// $data['Range'] = 'bytes='.$partStartPos.'-'.$partEndPos;
+// echo 'ranges';
+// }
+
+
+ if (isset($this->config['range']) or isset($this->config['multipartdownloadtype']) && $this->config['multipartdownloadtype'] == 'Range'){
$partEndPos = $partStartPos+self::PART_MIN_SIZE;
$data['Range'] = 'bytes='.$partStartPos.'-'.$partEndPos;
- echo 'ranges';
+ } else {
+ $data['PartNumber'] = $number;
}
-// if (isset($config['partNumber'])) {
-// $data['PartNumber'] = $config['partNumber'];
-// }
-
- $data['PartNumber'] = $number;
-
if (isset($config['add_content_md5'])
&& $config['add_content_md5'] === true
) {
@@ -150,11 +135,23 @@ public function setStreamPositionArray($sourceSize)
{
$parts = ceil($sourceSize/$this->state->getPartSize());
$position = 0;
- for ($i=1;$i<=$parts;$i++) {
- $this->streamPositionArray [$i]= $position;
- $position += $this->state->getPartSize();
+ if (isset($this->config['range']) or (isset($this->config['multipartdownloadtype']) && $this->config['multipartdownloadtype'] == 'Range')) {
+ for ($i = 1; $i <= $parts; $i++) {
+ $this->streamPositionArray [$position] = $i;
+ $position += $this->state->getPartSize();
+ }
+ } else {
+ for ($i = 1; $i <= $parts; $i++) {
+ $this->streamPositionArray [$i] = $position;
+ $position += $this->state->getPartSize();
+ }
}
-// print_r($this->streamPositionArray);
+// for ($i = 1; $i <= $parts; $i++) {
+// $this->streamPositionArray [$position] = $i;
+// $position += $this->state->getPartSize();
+// }
+
+ print_r($this->streamPositionArray);
}
protected function createDestStream($filePath)
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
index e87e77e462..b59d0c2749 100644
--- a/src/S3/MultipartDownloadingTrait.php
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -50,21 +50,46 @@ public static function getStateFromService(
return $state;
}
- protected function handleResult(CommandInterface $command, ResultInterface $result)
+ protected function handleResult($command, ResultInterface $result)
{
- $this->getState()->markPartAsUploaded($command['PartNumber'], [
- 'PartNumber' => $command['PartNumber'],
- 'ETag' => $this->extractETag($result),
- ]);
-
- $this->writeDestStream($command['PartNumber'], $result['Body']);
+ if (is_numeric($command)) {
+ $this->getState()->markPartAsUploaded($command, [
+ 'PartNumber' => $command,
+ 'ETag' => $this->extractETag($result),
+ ]);
+ $this->writeDestStream(1, $result['Body']);
+ } elseif (isset($command['PartNumber'])) {
+ $this->getState()->markPartAsUploaded($command['PartNumber'], [
+ 'PartNumber' => $command['PartNumber'],
+ 'ETag' => $this->extractETag($result),
+ ]);
+ $this->writeDestStream($command['PartNumber'], $result['Body']);
+ } else {
+ if (is_string($command)){
+ $seek = substr($command, strpos($command, "=") + 1);
+ } else {
+ $seek = substr($command['Range'], strpos($command['Range'], "=") + 1);
+ }
+ $seek = (int)(strtok($seek, '-'));
+ $this->getState()->markPartAsUploaded($this->streamPositionArray[$seek], [
+ 'PartNumber' => $this->streamPositionArray[$seek],
+ 'ETag' => $this->extractETag($result),
+ ]);
+ $this->writeDestStream($seek, $result['Body']);
+ }
}
protected function writeDestStream($partNum, $body)
{
- $bodyStream = Psr7\Utils::streamFor($body);
- $this->destStream->seek($this->streamPositionArray[$partNum]);
- $this->destStream->write($bodyStream->read(5242880));
+// echo "\n" . $body->getSize() . "\n";
+ if (isset($this->config['multipartdownloadtype']) && $this->config['multipartdownloadtype'] == 'Range' or isset($command['Range'])) {
+ $this->destStream->seek($partNum);
+ $this->destStream->write($body->getContents());
+ } else {
+ $this->destStream->seek($this->streamPositionArray[$partNum]);
+ $this->destStream->write($body->getContents());
+ }
+ echo "\n" . $this->destStream->getSize() . "\n";
}
abstract protected function extractETag(ResultInterface $result);
@@ -103,7 +128,7 @@ protected function determinePartSize()
return $partSize;
}
- protected function getInitiateParams()
+ protected function getInitiateParams($configType)
{
$config = $this->getConfig();
$params = isset($config['params']) ? $config['params'] : [];
@@ -112,25 +137,30 @@ protected function getInitiateParams()
$params['ACL'] = $config['acl'];
}
+ $params[$configType['config']] = $configType['configParam'];
+
+ return $params;
+ }
+
+ protected function getUploadType()
+ {
+ $config = $this->getConfig();
if (isset($config['partnumber'])) {
- $params['PartNumber'] = $config['partnumber'];
- echo 'PartNumber';
+ return ['config' => 'PartNumber',
+ 'configParam' => $config['partnumber']];
} elseif (isset($config['range'])) {
- $params['Range'] = $config['range'];
- echo 'Range';
+ return ['config' => 'Range',
+ 'configParam' => $config['range']];
} elseif (isset($config['multipartdownloadtype']) && $config['multipartdownloadtype'] == 'Range') {
- $params['Range'] = 'bytes=0-'.MultipartDownloader::PART_MIN_SIZE;
- echo 'multipartdownloadtype';
+ return ['config' => 'Range',
+ 'configParam' => 'bytes=1-'.MultipartDownloader::PART_MIN_SIZE,
+ 'type' => 'multi'
+ ];
} else {
- $params['PartNumber'] = 1;
- echo 'part num 1';
+ return ['config' => 'PartNumber',
+ 'configParam' => 1,
+ 'type' => 'multi'];
}
-
-// $params['PartNumber'] = $config['partnumber'];
-
-// $params['Range'] = 'bytes=0-1225';
-
- return $params;
}
// public function setStreamPosArray($sourceSize)
From 3169e59f9676db4594f1e3280a3bfba8358f9447 Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Tue, 13 Jun 2023 12:45:29 -0400
Subject: [PATCH 28/31] refactored handleresult & writedeststream
---
src/Multipart/AbstractDownloadManager.php | 3 +-
src/S3/MultipartDownloader.php | 14 --------
src/S3/MultipartDownloadingTrait.php | 44 ++++++++++-------------
3 files changed, 19 insertions(+), 42 deletions(-)
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
index 56b821f634..d260df7803 100644
--- a/src/Multipart/AbstractDownloadManager.php
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -109,7 +109,6 @@ public function promise()
}
$type = $this->getUploadType();
$result = (yield $this->execCommand('initiate', $this->getInitiateParams($type)));
- // if range or part config, end it here.
$this->determineSourceSize($result['ContentRange']);
$this->state->setUploadId(
$this->info['id']['upload_id'],
@@ -122,7 +121,7 @@ public function promise()
$this->handleResult($type['configParam'], $result);
}
}
-
+ // end download if PartNumber or Range is set, or object is only one part total.
if (isset($this->config['partnumber'])
or isset($this->config['range'])
or $result['PartsCount']==1){
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
index 5416c31130..51d5261043 100644
--- a/src/S3/MultipartDownloader.php
+++ b/src/S3/MultipartDownloader.php
@@ -102,14 +102,6 @@ protected function createPart($partStartPos, $number)
$data[$k] = $v;
}
-// } elseif (isset($config['multipartdownloadtype'])
-// && $config['multipartdownloadtype'] === 'Range') {
-// $partEndPos = $partStartPos+self::PART_MIN_SIZE;
-// $data['Range'] = 'bytes='.$partStartPos.'-'.$partEndPos;
-// echo 'ranges';
-// }
-
-
if (isset($this->config['range']) or isset($this->config['multipartdownloadtype']) && $this->config['multipartdownloadtype'] == 'Range'){
$partEndPos = $partStartPos+self::PART_MIN_SIZE;
$data['Range'] = 'bytes='.$partStartPos.'-'.$partEndPos;
@@ -146,12 +138,6 @@ public function setStreamPositionArray($sourceSize)
$position += $this->state->getPartSize();
}
}
-// for ($i = 1; $i <= $parts; $i++) {
-// $this->streamPositionArray [$position] = $i;
-// $position += $this->state->getPartSize();
-// }
-
- print_r($this->streamPositionArray);
}
protected function createDestStream($filePath)
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
index b59d0c2749..ee4f42d091 100644
--- a/src/S3/MultipartDownloadingTrait.php
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -52,44 +52,36 @@ public static function getStateFromService(
protected function handleResult($command, ResultInterface $result)
{
- if (is_numeric($command)) {
- $this->getState()->markPartAsUploaded($command, [
- 'PartNumber' => $command,
+ if (!($command instanceof CommandInterface)){
+ // single downloads - part/range
+ $this->getState()->markPartAsUploaded(1, [
+ 'PartNumber' => 1,
'ETag' => $this->extractETag($result),
]);
- $this->writeDestStream(1, $result['Body']);
- } elseif (isset($command['PartNumber'])) {
- $this->getState()->markPartAsUploaded($command['PartNumber'], [
- 'PartNumber' => $command['PartNumber'],
- 'ETag' => $this->extractETag($result),
- ]);
- $this->writeDestStream($command['PartNumber'], $result['Body']);
- } else {
- if (is_string($command)){
- $seek = substr($command, strpos($command, "=") + 1);
- } else {
- $seek = substr($command['Range'], strpos($command['Range'], "=") + 1);
- }
+ $this->writeDestStream(0, $result['Body']);
+ } elseif (!(isset($command['PartNumber']))) {
+ // multi downloads - range
+ $seek = substr($command['Range'], strpos($command['Range'], "=") + 1);
$seek = (int)(strtok($seek, '-'));
$this->getState()->markPartAsUploaded($this->streamPositionArray[$seek], [
'PartNumber' => $this->streamPositionArray[$seek],
'ETag' => $this->extractETag($result),
]);
$this->writeDestStream($seek, $result['Body']);
+ } else {
+ // multi downloads - part
+ $this->getState()->markPartAsUploaded($command['PartNumber'], [
+ 'PartNumber' => $command['PartNumber'],
+ 'ETag' => $this->extractETag($result),
+ ]);
+ $this->writeDestStream($this->streamPositionArray[$command['PartNumber']], $result['Body']);
}
}
protected function writeDestStream($partNum, $body)
{
-// echo "\n" . $body->getSize() . "\n";
- if (isset($this->config['multipartdownloadtype']) && $this->config['multipartdownloadtype'] == 'Range' or isset($command['Range'])) {
- $this->destStream->seek($partNum);
- $this->destStream->write($body->getContents());
- } else {
- $this->destStream->seek($this->streamPositionArray[$partNum]);
- $this->destStream->write($body->getContents());
- }
- echo "\n" . $this->destStream->getSize() . "\n";
+ $this->destStream->seek($partNum);
+ $this->destStream->write($body->getContents());
}
abstract protected function extractETag(ResultInterface $result);
@@ -153,7 +145,7 @@ protected function getUploadType()
'configParam' => $config['range']];
} elseif (isset($config['multipartdownloadtype']) && $config['multipartdownloadtype'] == 'Range') {
return ['config' => 'Range',
- 'configParam' => 'bytes=1-'.MultipartDownloader::PART_MIN_SIZE,
+ 'configParam' => 'bytes=0-'.MultipartDownloader::PART_MIN_SIZE,
'type' => 'multi'
];
} else {
From a46b3be1527b0d2dfb2068c177251f0d9fac2dba Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 19 Jun 2023 15:11:33 -0400
Subject: [PATCH 29/31] beginning tests
---
src/Multipart/AbstractDownloadManager.php | 2 +-
src/Multipart/DownloadState.php | 14 +-
src/S3/MultipartDownloader.php | 12 +-
src/S3/MultipartDownloadingTrait.php | 18 +-
tests/Multipart/AbstractDownloaderTest.php | 250 +++++++++++++
tests/Multipart/DownloadStateTest.php | 232 ++++++++++++
.../S3MultipartDownloadExceptionTest.php | 41 ++
tests/S3/MultipartDownloaderTest.php | 349 +++++++++++++++++-
8 files changed, 897 insertions(+), 21 deletions(-)
create mode 100644 tests/Multipart/AbstractDownloaderTest.php
create mode 100644 tests/Multipart/DownloadStateTest.php
create mode 100644 tests/S3/Exception/S3MultipartDownloadExceptionTest.php
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
index d260df7803..1787ca2d69 100644
--- a/src/Multipart/AbstractDownloadManager.php
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -115,7 +115,7 @@ public function promise()
$result[$this->info['id']['upload_id']]
);
$this->state->setStatus(DownloadState::INITIATED);
- if (isset($type['type'])){
+ if (isset($type['multipart'])){
$this->handleResult(1, $result);
} else {
$this->handleResult($type['configParam'], $result);
diff --git a/src/Multipart/DownloadState.php b/src/Multipart/DownloadState.php
index 6fa28016b7..aaa9782985 100644
--- a/src/Multipart/DownloadState.php
+++ b/src/Multipart/DownloadState.php
@@ -20,7 +20,7 @@ class DownloadState
private $partSize;
/** @var array Parts that have been uploaded. */
- private $uploadedParts = [];
+ private $downloadedParts = [];
/** @var int Identifies the status the upload. */
private $status = self::CREATED;
@@ -84,9 +84,9 @@ public function setPartSize($partSize)
* @param array $partData Data from the upload operation that needs to be
* recalled during the complete operation.
*/
- public function markPartAsUploaded($partNumber, array $partData = [])
+ public function markPartAsDownloaded($partNumber, array $partData = [])
{
- $this->uploadedParts[$partNumber] = $partData;
+ $this->downloadedParts[$partNumber] = $partData;
}
/**
@@ -98,7 +98,7 @@ public function markPartAsUploaded($partNumber, array $partData = [])
*/
public function hasPartBeenUploaded($partNumber)
{
- return isset($this->uploadedParts[$partNumber]);
+ return isset($this->downloadedParts[$partNumber]);
}
/**
@@ -106,10 +106,10 @@ public function hasPartBeenUploaded($partNumber)
*
* @return array
*/
- public function getUploadedParts()
+ public function getDownloadedParts()
{
- ksort($this->uploadedParts);
- return $this->uploadedParts;
+ ksort($this->downloadedParts);
+ return $this->downloadedParts;
}
/**
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
index 51d5261043..89d7439279 100644
--- a/src/S3/MultipartDownloader.php
+++ b/src/S3/MultipartDownloader.php
@@ -20,6 +20,9 @@ class MultipartDownloader extends AbstractDownloader
const PART_MAX_SIZE = 5368709120;
const PART_MAX_NUM = 10000;
+ public $destStream;
+ public $streamPositionArray;
+
/**
* Creates a multipart download for an S3 object.
*
@@ -102,7 +105,10 @@ protected function createPart($partStartPos, $number)
$data[$k] = $v;
}
- if (isset($this->config['range']) or isset($this->config['multipartdownloadtype']) && $this->config['multipartdownloadtype'] == 'Range'){
+ // Set Range or PartNumber params
+ if (isset($this->config['range']) or
+ isset($this->config['multipartdownloadtype'])
+ && $this->config['multipartdownloadtype'] == 'Range'){
$partEndPos = $partStartPos+self::PART_MIN_SIZE;
$data['Range'] = 'bytes='.$partStartPos.'-'.$partEndPos;
} else {
@@ -127,7 +133,9 @@ public function setStreamPositionArray($sourceSize)
{
$parts = ceil($sourceSize/$this->state->getPartSize());
$position = 0;
- if (isset($this->config['range']) or (isset($this->config['multipartdownloadtype']) && $this->config['multipartdownloadtype'] == 'Range')) {
+ if (isset($this->config['range']) or
+ (isset($this->config['multipartdownloadtype']) &&
+ $this->config['multipartdownloadtype'] == 'Range')) {
for ($i = 1; $i <= $parts; $i++) {
$this->streamPositionArray [$position] = $i;
$position += $this->state->getPartSize();
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
index ee4f42d091..7235299576 100644
--- a/src/S3/MultipartDownloadingTrait.php
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -38,7 +38,7 @@ public static function getStateFromService(
}
// Mark all the parts returned by ListParts as uploaded.
foreach ($result['Parts'] as $part) {
- $state->markPartAsUploaded($part['PartNumber'], [
+ $state->markPartAsDownloaded($part['PartNumber'], [
'PartNumber' => $part['PartNumber'],
'ETag' => $part['ETag']
]);
@@ -54,7 +54,7 @@ protected function handleResult($command, ResultInterface $result)
{
if (!($command instanceof CommandInterface)){
// single downloads - part/range
- $this->getState()->markPartAsUploaded(1, [
+ $this->getState()->markPartAsDownloaded(1, [
'PartNumber' => 1,
'ETag' => $this->extractETag($result),
]);
@@ -63,14 +63,14 @@ protected function handleResult($command, ResultInterface $result)
// multi downloads - range
$seek = substr($command['Range'], strpos($command['Range'], "=") + 1);
$seek = (int)(strtok($seek, '-'));
- $this->getState()->markPartAsUploaded($this->streamPositionArray[$seek], [
+ $this->getState()->markPartAsDownloaded($this->streamPositionArray[$seek], [
'PartNumber' => $this->streamPositionArray[$seek],
'ETag' => $this->extractETag($result),
]);
$this->writeDestStream($seek, $result['Body']);
} else {
// multi downloads - part
- $this->getState()->markPartAsUploaded($command['PartNumber'], [
+ $this->getState()->markPartAsDownloaded($command['PartNumber'], [
'PartNumber' => $command['PartNumber'],
'ETag' => $this->extractETag($result),
]);
@@ -78,9 +78,9 @@ protected function handleResult($command, ResultInterface $result)
}
}
- protected function writeDestStream($partNum, $body)
+ protected function writeDestStream($position, $body)
{
- $this->destStream->seek($partNum);
+ $this->destStream->seek($position);
$this->destStream->write($body->getContents());
}
@@ -92,7 +92,7 @@ protected function getCompleteParams()
$params = isset($config['params']) ? $config['params'] : [];
$params['MultipartUpload'] = [
- 'Parts' => $this->getState()->getUploadedParts()
+ 'Parts' => $this->getState()->getDownloadedParts()
];
return $params;
@@ -146,12 +146,12 @@ protected function getUploadType()
} elseif (isset($config['multipartdownloadtype']) && $config['multipartdownloadtype'] == 'Range') {
return ['config' => 'Range',
'configParam' => 'bytes=0-'.MultipartDownloader::PART_MIN_SIZE,
- 'type' => 'multi'
+ 'multipart' => 'yes'
];
} else {
return ['config' => 'PartNumber',
'configParam' => 1,
- 'type' => 'multi'];
+ 'multipart' => 'yes'];
}
}
diff --git a/tests/Multipart/AbstractDownloaderTest.php b/tests/Multipart/AbstractDownloaderTest.php
new file mode 100644
index 0000000000..e1968343e9
--- /dev/null
+++ b/tests/Multipart/AbstractDownloaderTest.php
@@ -0,0 +1,250 @@
+ 'foo', 'Key' => 'bar']);
+ $state->setPartSize(2);
+ $state->setStatus($status);
+
+ return $this->getTestDownloader(
+ $source ?: Psr7\Utils::streamFor(),
+ ['state' => $state],
+ $results
+ );
+ }
+
+ private function getTestDownloader(
+ $source = null,
+ array $config = [],
+ array $results = []
+ ) {
+ $client = $this->getTestClient('s3', [
+ 'validate' => false,
+ 'retries' => 0,
+ ]);
+ $this->addMockResults($client, $results);
+
+ return new TestDownloader($client, $source ?: Psr7\Utils::streamFor(), $config);
+ }
+
+ public function testThrowsExceptionOnBadInitiateRequest()
+ {
+ $this->expectException(\Aws\S3\Exception\S3MultipartDownloadException::class);
+ $downloader = $this->getDownloaderWithState(DownloadState::CREATED, [
+ new AwsException('Failed', new Command('Initiate')),
+ ]);
+ $downloader->download();
+ }
+
+ public function testThrowsExceptionIfStateIsCompleted()
+ {
+ $this->expectException(\LogicException::class);
+ $uploader = $this->getUploaderWithState(UploadState::COMPLETED);
+ $this->assertTrue($uploader->getState()->isCompleted());
+ $uploader->upload();
+ }
+
+ public function testSuccessfulCompleteReturnsResult()
+ {
+ $uploader = $this->getUploaderWithState(UploadState::CREATED, [
+ new Result(), // Initiate
+ new Result(), // Upload
+ new Result(), // Upload
+ new Result(), // Upload
+ new Result(['test' => 'foo']) // Complete
+ ], Psr7\Utils::streamFor('abcdef'));
+ $this->assertSame('foo', $uploader->upload()['test']);
+ $this->assertTrue($uploader->getState()->isCompleted());
+ }
+
+ public function testThrowsExceptionOnBadCompleteRequest()
+ {
+ $this->expectException(\Aws\S3\Exception\S3MultipartUploadException::class);
+ $uploader = $this->getUploaderWithState(UploadState::CREATED, [
+ new Result(), // Initiate
+ new Result(), // Upload
+ new AwsException('Failed', new Command('Complete')),
+ ], Psr7\Utils::streamFor('a'));
+ $uploader->upload();
+ }
+
+ public function testThrowsExceptionOnBadUploadRequest()
+ {
+ $uploader = $this->getUploaderWithState(UploadState::CREATED, [
+ new Result(), // Initiate
+ new AwsException('Failed[1]', new Command('Upload', ['PartNumber' => 1])),
+ new Result(), // Upload
+ new Result(), // Upload
+ new AwsException('Failed[4]', new Command('Upload', ['PartNumber' => 4])),
+ new Result(), // Upload
+ ], Psr7\Utils::streamFor('abcdefghi'));
+
+ try {
+ $uploader->upload();
+ $this->fail('No exception was thrown.');
+ } catch (MultipartUploadException $e) {
+ $message = $e->getMessage();
+ $this->assertStringContainsString('Failed[1]', $message);
+ $this->assertStringContainsString('Failed[4]', $message);
+ $uploadedParts = $e->getState()->getDownloadedParts();
+ $this->assertCount(3, $uploadedParts);
+ $this->assertArrayHasKey(2, $uploadedParts);
+ $this->assertArrayHasKey(3, $uploadedParts);
+ $this->assertArrayHasKey(5, $uploadedParts);
+
+ // Test if can resume an upload.
+ $serializedState = serialize($e->getState());
+ $state = unserialize($serializedState);
+ $secondChance = $this->getTestUploader(
+ Psr7\Utils::streamFor('abcdefghi'),
+ ['state' => $state],
+ [
+ new Result(), // Upload
+ new Result(), // Upload
+ new Result(['foo' => 'bar']), // Upload
+ ]
+ );
+ $result = $secondChance->upload();
+ $this->assertSame('bar', $result['foo']);
+ }
+ }
+
+ public function testAsyncUpload()
+ {
+ $called = 0;
+ $fn = function () use (&$called) {
+ $called++;
+ };
+
+ $uploader = $this->getTestUploader(Psr7\Utils::streamFor('abcde'), [
+ 'bucket' => 'foo',
+ 'key' => 'bar',
+ 'prepare_data_source' => $fn,
+ 'before_initiate' => $fn,
+ 'before_upload' => $fn,
+ 'before_complete' => $fn,
+ ], [
+ new Result(), // Initiate
+ new Result(), // Upload
+ new Result(), // Upload
+ new Result(), // Upload
+ new Result(['test' => 'foo']) // Complete
+ ]);
+
+ $promise = $uploader->promise();
+ $this->assertSame($promise, $uploader->promise());
+ $this->assertInstanceOf('Aws\Result', $promise->wait());
+ $this->assertSame(6, $called);
+ }
+
+ public function testRequiresIdParams()
+ {
+ $this->expectException(\InvalidArgumentException::class);
+ $this->getTestUploader(Psr7\Utils::streamFor());
+ }
+
+ public function testCanSetSourceFromFilenameIfExists()
+ {
+ $config = ['bucket' => 'foo', 'key' => 'bar'];
+
+ // CASE 1: Filename exists.
+ $uploader = $this->getTestUploader(__FILE__, $config);
+ $this->assertInstanceOf(
+ 'Psr\Http\Message\StreamInterface',
+ $this->getPropertyValue($uploader, 'source')
+ );
+
+ // CASE 2: Filename does not exist.
+ $exception = null;
+ try {
+ $this->getTestUploader('non-existent-file.foobar', $config);
+ } catch (\Exception $exception) {}
+ $this->assertInstanceOf('RuntimeException', $exception);
+
+ // CASE 3: Source stream is not readable.
+ $exception = null;
+ try {
+ $this->getTestUploader(STDERR, $config);
+ } catch (\Exception $exception) {}
+ $this->assertInstanceOf('InvalidArgumentException', $exception);
+ }
+
+ /**
+ * @param bool $seekable
+ * @param UploadState $state
+ * @param array $expectedBodies
+ *
+ * @dataProvider getPartGeneratorTestCases
+ */
+ public function testCommandGeneratorYieldsExpectedUploadCommands(
+ $seekable,
+ UploadState $state,
+ array $expectedBodies
+ ) {
+ $source = Psr7\Utils::streamFor(fopen(__DIR__ . '/source.txt', 'r'));
+ if (!$seekable) {
+ $source = new Psr7\NoSeekStream($source);
+ }
+
+ $uploader = $this->getTestUploader($source, ['state' => $state]);
+ $uploader->getState();
+ $handler = function (callable $handler) {
+ return function ($c, $r) use ($handler) {
+ return $handler($c, $r);
+ };
+ };
+
+ $actualBodies = [];
+ $getUploadCommands = (new \ReflectionObject($uploader))
+ ->getMethod('getUploadCommands');
+ $getUploadCommands->setAccessible(true);
+ foreach ($getUploadCommands->invoke($uploader, $handler) as $cmd) {
+ $actualBodies[$cmd['PartNumber']] = $cmd['Body']->getContents();
+ }
+
+ $this->assertEquals($expectedBodies, $actualBodies);
+ }
+
+ public function getPartGeneratorTestCases()
+ {
+ $expected = [
+ 1 => 'AA',
+ 2 => 'BB',
+ 3 => 'CC',
+ 4 => 'DD',
+ 5 => 'EE',
+ 6 => 'F' ,
+ ];
+ $expectedSkip = $expected;
+ unset($expectedSkip[1], $expectedSkip[2], $expectedSkip[4]);
+ $state = new UploadState([]);
+ $state->setPartSize(2);
+ $stateSkip = clone $state;
+ $stateSkip->markPartAsUploaded(1);
+ $stateSkip->markPartAsUploaded(2);
+ $stateSkip->markPartAsUploaded(4);
+ return [
+ [true, $state, $expected],
+ [false, $state, $expected],
+ [true, $stateSkip, $expectedSkip],
+ [false, $stateSkip, $expectedSkip],
+ ];
+ }
+}
diff --git a/tests/Multipart/DownloadStateTest.php b/tests/Multipart/DownloadStateTest.php
new file mode 100644
index 0000000000..3ccfaefaae
--- /dev/null
+++ b/tests/Multipart/DownloadStateTest.php
@@ -0,0 +1,232 @@
+ true]);
+ $this->assertArrayHasKey('a', $state->getId());
+ // Note: the state should not be initiated at first.
+ $this->assertFalse($state->isInitiated());
+ $this->assertFalse($state->isCompleted());
+
+ $state->setUploadId('b', true);
+ $this->assertArrayHasKey('b', $state->getId());
+ $this->assertArrayHasKey('a', $state->getId());
+
+ $state->setStatus(DownloadState::INITIATED);
+ $this->assertFalse($state->isCompleted());
+ $this->assertTrue($state->isInitiated());
+
+ $state->setStatus(DownloadState::COMPLETED);
+ $this->assertFalse($state->isInitiated());
+ $this->assertTrue($state->isCompleted());
+ }
+
+ public function testCanStorePartSize()
+ {
+ $state = new DownloadState([]);
+ $this->assertNull($state->getPartSize());
+ $state->setPartSize(50000000);
+ $this->assertSame(50000000, $state->getPartSize());
+ }
+
+ public function testCanTrackDownloadedParts()
+ {
+ $state = new DownloadState([]);
+ $this->assertEmpty($state->getDownloadedParts());
+
+ $state->markPartAsUploaded(1, ['foo' => 1]);
+ $state->markPartAsUploaded(3, ['foo' => 3]);
+ $state->markPartAsUploaded(2, ['foo' => 2]);
+
+ $this->assertTrue($state->hasPartBeenUploaded(2));
+ $this->assertFalse($state->hasPartBeenUploaded(5));
+
+ // Note: The parts should come out sorted.
+ $this->assertSame([1, 2, 3], array_keys($state->getDownloadedParts()));
+ }
+
+ public function testSerializationWorks()
+ {
+ $state = new DownloadState([]);
+ $state->setPartSize(5);
+ $state->markPartAsDownloaded(1);
+ $state->setStatus($state::INITIATED);
+ $state->setUploadId('foo', 'bar');
+ $serializedState = serialize($state);
+
+ /** @var DownloadState $newState */
+ $newState = unserialize($serializedState);
+ $this->assertSame(5, $newState->getPartSize());
+ $this->assertArrayHasKey(1, $state->getDownloadedParts());
+ $this->assertTrue($newState->isInitiated());
+ $this->assertArrayHasKey('foo', $newState->getId());
+ }
+
+// public function testEmptyUploadStateOutputWithConfigFalse()
+// {
+// $config['track_upload'] = false;
+// $state = new DownloadState([], $config);
+// $state->setProgressThresholds(100);
+// $state->displayProgress(13);
+// $this->expectOutputString('');
+// }
+
+ /**
+ * @dataProvider getDisplayProgressCases
+ */
+// public function testDisplayProgressPrintsProgress(
+// $totalSize,
+// $totalUploaded,
+// $progressBar
+// ) {
+// $config['track_upload'] = true;
+// $state = new UploadState([]);
+// $state->setProgressThresholds($totalSize);
+// $state->displayProgress($totalUploaded);
+//
+// $this->expectOutputString($progressBar);
+// }
+
+// public function getDisplayProgressCases()
+// {
+// $progressBar = ["Transfer initiated...\n| | 0.0%\n",
+// "|== | 12.5%\n",
+// "|===== | 25.0%\n",
+// "|======= | 37.5%\n",
+// "|========== | 50.0%\n",
+// "|============ | 62.5%\n",
+// "|=============== | 75.0%\n",
+// "|================= | 87.5%\n",
+// "|====================| 100.0%\nTransfer complete!\n"];
+// return [
+// [100000, 0, $progressBar[0]],
+// [100000, 12499, $progressBar[0]],
+// [100000, 12500, "{$progressBar[0]}{$progressBar[1]}"],
+// [100000, 24999, "{$progressBar[0]}{$progressBar[1]}"],
+// [100000, 25000, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"],
+// [100000, 37499, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"],
+// [
+// 100000,
+// 37500,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}"
+// ],
+// [
+// 100000,
+// 49999,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}"
+// ],
+// [
+// 100000,
+// 50000,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}"
+// ],
+// [
+// 100000,
+// 62499,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}"
+// ],
+// [
+// 100000,
+// 62500,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+// "{$progressBar[5]}"
+// ],
+// [
+// 100000,
+// 74999,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+// "{$progressBar[5]}"
+// ],
+// [
+// 100000,
+// 75000,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+// "{$progressBar[5]}{$progressBar[6]}"
+// ],
+// [
+// 100000,
+// 87499,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+// "{$progressBar[5]}{$progressBar[6]}"
+// ],
+// [
+// 100000,
+// 87500,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+// "{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}"
+// ],
+// [
+// 100000,
+// 99999,
+// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
+// "{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}"
+// ],
+// [100000, 100000, implode($progressBar)]
+// ];
+// }
+//
+// /**
+// * @dataProvider getThresholdCases
+// */
+// public function testUploadThresholds($totalSize)
+// {
+// $config['track_upload'] = true;
+// $state = new UploadState([]);
+// $threshold = $state->setProgressThresholds($totalSize);
+//
+// $this->assertIsArray($threshold);
+// $this->assertCount(9, $threshold);
+// }
+//
+// public function getThresholdCases()
+// {
+// return [
+// [0],
+// [100000],
+// [100001]
+// ];
+// }
+//
+// /**
+// * @dataProvider getInvalidIntCases
+// */
+// public function testSetProgressThresholdsThrowsException($totalSize)
+// {
+// $state = new UploadState([]);
+// $this->expectExceptionMessage('The total size of the upload must be an int.');
+// $this->expectException(\InvalidArgumentException::class);
+//
+// $state->setProgressThresholds($totalSize);
+// }
+//
+// /**
+// * @dataProvider getInvalidIntCases
+// */
+// public function testDisplayProgressThrowsException($totalUploaded)
+// {
+// $state = new UploadState([]);
+// $this->expectExceptionMessage('The size of the bytes being uploaded must be an int.');
+// $this->expectException(\InvalidArgumentException::class);
+//
+// $state->displayProgress($totalUploaded);
+// }
+//
+// public function getInvalidIntCases()
+// {
+// return [
+// [''],
+// [null],
+// ['1234'],
+// ['aws'],
+// ];
+// }
+}
\ No newline at end of file
diff --git a/tests/S3/Exception/S3MultipartDownloadExceptionTest.php b/tests/S3/Exception/S3MultipartDownloadExceptionTest.php
new file mode 100644
index 0000000000..c7df556c52
--- /dev/null
+++ b/tests/S3/Exception/S3MultipartDownloadExceptionTest.php
@@ -0,0 +1,41 @@
+ new AwsException('Bad digest.', new Command('GetObject', [
+ 'Bucket' => 'foo',
+ 'Key' => 'bar'
+// 'Body' => Psr7\Utils::streamFor('Part 1'),
+ ])),
+ 5 => new AwsException('Missing header.', new Command('GetObject', [
+ 'Bucket' => 'foo',
+ 'Key' => 'bar',
+// 'Body' => Psr7\Utils::streamFor('Part 2'),
+ ])),
+ 8 => new AwsException('Needs more love.', new Command('GetObject')),
+ ];
+
+ $path = '/path/to/the/large/file/test.zip';
+ $exception = new S3MultipartDownloadException($state, $failed, [
+ 'file_name' => $path
+ ]);
+ $this->assertSame('foo', $exception->getBucket());
+ $this->assertSame('bar', $exception->getKey());
+ $this->assertSame('php://temp', $exception->getSourceFileName());
+ }
+}
diff --git a/tests/S3/MultipartDownloaderTest.php b/tests/S3/MultipartDownloaderTest.php
index c22e072110..289e35b322 100644
--- a/tests/S3/MultipartDownloaderTest.php
+++ b/tests/S3/MultipartDownloaderTest.php
@@ -12,9 +12,354 @@
use Yoast\PHPUnitPolyfills\TestCases\TestCase;
/**
-* @covers Aws\S3\MultipartDownloader
-*/
+ * @covers Aws\S3\MultipartDownloader
+ */
class MultipartDownloaderTest extends TestCase
{
+ use UsesServiceTrait;
+ const MB = 1048576;
+ const FILENAME = '_aws-sdk-php-s3-mup-test-dots.txt';
+
+ public static function tear_down_after_class()
+ {
+ @unlink(sys_get_temp_dir() . '/' . self::FILENAME);
+ }
+
+ /**
+ * @dataProvider getTestCases
+ */
+ public function testS3MultipartDownloadWorkflow(
+ array $clientOptions = [],
+ array $uploadOptions = [],
+ $dest = null,
+ $error = false
+ ) {
+ $client = $this->getTestClient('s3', $clientOptions);
+ $url = 'http://foo.s3.amazonaws.com/bar';
+ $this->addMockResults($client, [
+ new Result(['UploadId' => 'baz']),
+ new Result(['ETag' => 'A']),
+ new Result(['ETag' => 'B']),
+ new Result(['ETag' => 'C']),
+ new Result(['Location' => $url])
+ ]);
+
+ if ($error) {
+ if (method_exists($this, 'expectException')) {
+ $this->expectException($error);
+ } else {
+ $this->setExpectedException($error);
+ }
+ }
+
+ $uploader = new MultipartDownloader($client, $dest, $uploadOptions);
+ $result = $uploader->download();
+
+ $this->assertTrue($uploader->getState()->isCompleted());
+ $this->assertSame($url, $result['ObjectURL']);
+ }
+
+ public function getTestCases()
+ {
+ $defaults = [
+ 'bucket' => 'foo',
+ 'key' => 'bar',
+ ];
+
+ $data = str_repeat('.', 12 * self::MB);
+ $filename = sys_get_temp_dir() . '/' . self::FILENAME;
+ file_put_contents($filename, $data);
+
+ return [
+ [ // Seekable stream, regular config
+ [],
+ ['acl' => 'private'] + $defaults,
+ Psr7\Utils::streamFor(fopen($filename, 'r'))
+ ],
+ [ // Non-seekable stream
+ [],
+ $defaults,
+ Psr7\Utils::streamFor($data)
+ ],
+ [ // Error: bad part_size
+ [],
+ ['part_size' => 1] + $defaults,
+ Psr7\FnStream::decorate(
+ Psr7\Utils::streamFor($data), [
+ 'getSize' => function () {return null;}
+ ]
+ ),
+ 'InvalidArgumentException'
+ ],
+ ];
+ }
+
+ public function testCanLoadStateFromService()
+ {
+ $client = $this->getTestClient('s3');
+ $url = 'http://foo.s3.amazonaws.com/bar';
+ $this->addMockResults($client, [
+ new Result(['Parts' => [
+ ['PartNumber' => 1, 'ETag' => 'A', 'Size' => 4 * self::MB],
+ ]]),
+ new Result(['ETag' => 'B']),
+ new Result(['ETag' => 'C']),
+ new Result(['Location' => $url])
+ ]);
+
+ $state = MultipartUploader::getStateFromService($client, 'foo', 'bar', 'baz');
+ $source = Psr7\Utils::streamFor(str_repeat('.', 9 * self::MB));
+ $uploader = new MultipartUploader($client, $source, ['state' => $state]);
+ $result = $uploader->upload();
+
+ $this->assertTrue($uploader->getState()->isCompleted());
+ $this->assertSame(4 * self::MB, $uploader->getState()->getPartSize());
+ $this->assertSame($url, $result['ObjectURL']);
+ }
+
+ public function testCanUseCaseInsensitiveConfigKeys()
+ {
+ $client = $this->getTestClient('s3');
+ $putObjectMup = new MultipartUploader($client, Psr7\Utils::streamFor('x'), [
+ 'Bucket' => 'bucket',
+ 'Key' => 'key',
+ ]);
+ $classicMup = new MultipartUploader($client, Psr7\Utils::streamFor('x'), [
+ 'bucket' => 'bucket',
+ 'key' => 'key',
+ ]);
+ $configProp = (new \ReflectionClass(MultipartUploader::class))
+ ->getProperty('config');
+ $configProp->setAccessible(true);
+
+ $this->assertSame($configProp->getValue($classicMup), $configProp->getValue($putObjectMup));
+ }
+
+ /** @doesNotPerformAssertions */
+ public function testMultipartSuccessStreams()
+ {
+ $size = 12 * self::MB;
+ $data = str_repeat('.', $size);
+ $filename = sys_get_temp_dir() . '/' . self::FILENAME;
+ file_put_contents($filename, $data);
+
+ return [
+ [ // Seekable stream, regular config
+ Psr7\Utils::streamFor(fopen($filename, 'r')),
+ $size,
+ ],
+ [ // Non-seekable stream
+ Psr7\Utils::streamFor($data),
+ $size,
+ ]
+ ];
+ }
+
+ /**
+ * @dataProvider testMultipartSuccessStreams
+ */
+ public function testS3MultipartUploadParams($stream, $size)
+ {
+ /** @var \Aws\S3\S3Client $client */
+ $client = $this->getTestClient('s3');
+ $client->getHandlerList()->appendSign(
+ Middleware::tap(function ($cmd, $req) {
+ $name = $cmd->getName();
+ if ($name === 'UploadPart') {
+ $this->assertTrue(
+ $req->hasHeader('Content-MD5')
+ );
+ }
+ })
+ );
+ $uploadOptions = [
+ 'bucket' => 'foo',
+ 'key' => 'bar',
+ 'add_content_md5' => true,
+ 'params' => [
+ 'RequestPayer' => 'test',
+ 'ContentLength' => $size
+ ],
+ 'before_initiate' => function($command) {
+ $this->assertSame('test', $command['RequestPayer']);
+ },
+ 'before_upload' => function($command) use ($size) {
+ $this->assertLessThan($size, $command['ContentLength']);
+ $this->assertSame('test', $command['RequestPayer']);
+ },
+ 'before_complete' => function($command) {
+ $this->assertSame('test', $command['RequestPayer']);
+ }
+ ];
+ $url = 'http://foo.s3.amazonaws.com/bar';
+
+ $this->addMockResults($client, [
+ new Result(['UploadId' => 'baz']),
+ new Result(['ETag' => 'A']),
+ new Result(['ETag' => 'B']),
+ new Result(['ETag' => 'C']),
+ new Result(['Location' => $url])
+ ]);
+
+ $uploader = new MultipartUploader($client, $stream, $uploadOptions);
+ $result = $uploader->upload();
+
+ $this->assertTrue($uploader->getState()->isCompleted());
+ $this->assertSame($url, $result['ObjectURL']);
+ }
+
+ public function getContentTypeSettingTests()
+ {
+ $size = 12 * self::MB;
+ $data = str_repeat('.', $size);
+ $filename = sys_get_temp_dir() . '/' . self::FILENAME;
+ file_put_contents($filename, $data);
+
+ return [
+ [ // Successful lookup from filename via stream
+ Psr7\Utils::streamFor(fopen($filename, 'r')),
+ [],
+ 'text/plain'
+ ],
+ [ // Unsuccessful lookup because of no file name
+ Psr7\Utils::streamFor($data),
+ [],
+ 'application/octet-stream'
+ ],
+ [ // Successful override of known type from filename
+ Psr7\Utils::streamFor(fopen($filename, 'r')),
+ ['ContentType' => 'TestType'],
+ 'TestType'
+ ],
+ [ // Successful override of unknown type
+ Psr7\Utils::streamFor($data),
+ ['ContentType' => 'TestType'],
+ 'TestType'
+ ]
+ ];
+ }
+
+ /**
+ * @dataProvider getContentTypeSettingTests
+ */
+ public function testS3MultipartContentTypeSetting(
+ $stream,
+ $params,
+ $expectedContentType
+ ) {
+ /** @var \Aws\S3\S3Client $client */
+ $client = $this->getTestClient('s3');
+ $uploadOptions = [
+ 'bucket' => 'foo',
+ 'key' => 'bar',
+ 'params' => $params,
+ 'before_initiate' => function($command) use ($expectedContentType) {
+ $this->assertEquals(
+ $expectedContentType,
+ $command['ContentType']
+ );
+ },
+ ];
+ $url = 'http://foo.s3.amazonaws.com/bar';
+
+ $this->addMockResults($client, [
+ new Result(['UploadId' => 'baz']),
+ new Result(['ETag' => 'A']),
+ new Result(['ETag' => 'B']),
+ new Result(['ETag' => 'C']),
+ new Result(['Location' => $url])
+ ]);
+
+ $uploader = new MultipartUploader($client, $stream, $uploadOptions);
+ $result = $uploader->upload();
+
+ $this->assertTrue($uploader->getState()->isCompleted());
+ $this->assertSame($url, $result['ObjectURL']);
+ }
+
+ public function testAppliesAmbiguousSuccessParsing()
+ {
+ $this->expectExceptionMessage("An exception occurred while uploading parts to a multipart upload");
+ $this->expectException(\Aws\S3\Exception\S3MultipartUploadException::class);
+ $counter = 0;
+
+ $httpHandler = function ($request, array $options) use (&$counter) {
+ if ($counter < 1) {
+ $body = "baz";
+ } else {
+ $body = "\n\n\n";
+ }
+ $counter++;
+
+ return Promise\Create::promiseFor(
+ new Psr7\Response(200, [], $body)
+ );
+ };
+
+ $s3 = new S3Client([
+ 'version' => 'latest',
+ 'region' => 'us-east-1',
+ 'http_handler' => $httpHandler
+ ]);
+
+ $data = str_repeat('.', 12 * 1048576);
+ $source = Psr7\Utils::streamFor($data);
+
+ $uploader = new MultipartUploader(
+ $s3,
+ $source,
+ [
+ 'bucket' => 'test-bucket',
+ 'key' => 'test-key'
+ ]
+ );
+ $uploader->upload();
+ }
+
+ public function testFailedUploadPrintsPartialProgressBar()
+ {
+ $partialBar = [ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n"];
+ $this->expectOutputString("{$partialBar[0]}{$partialBar[1]}{$partialBar[2]}");
+
+ $this->expectExceptionMessage("An exception occurred while uploading parts to a multipart upload");
+ $this->expectException(\Aws\S3\Exception\S3MultipartUploadException::class);
+ $counter = 0;
+
+ $httpHandler = function ($request, array $options) use (&$counter) {
+ if ($counter < 4) {
+ $body = "baz";
+ } else {
+ $body = "\n\n\n";
+ }
+ $counter++;
+
+ return Promise\Create::promiseFor(
+ new Psr7\Response(200, [], $body)
+ );
+ };
+
+ $s3 = new S3Client([
+ 'version' => 'latest',
+ 'region' => 'us-east-1',
+ 'http_handler' => $httpHandler
+ ]);
+
+ $data = str_repeat('.', 50 * self::MB);
+ $source = Psr7\Utils::streamFor($data);
+
+ $uploader = new MultipartUploader(
+ $s3,
+ $source,
+ [
+ 'bucket' => 'test-bucket',
+ 'key' => 'test-key',
+ 'track_upload' => 'true'
+ ]
+ );
+ $uploader->upload();
+ }
}
+
From 8d00990ad2f997abfb3feaced42320e1cbe32d0a Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 19 Jun 2023 23:49:01 -0400
Subject: [PATCH 30/31] Refactoring downloader, unfinished tests
---
src/Multipart/AbstractDownloadManager.php | 99 +++++-----
src/Multipart/AbstractDownloader.php | 57 +++---
src/Multipart/DownloadState.php | 49 +++--
src/S3/MultipartDownloader.php | 63 ++++---
src/S3/MultipartDownloadingTrait.php | 89 +++------
.../MultipartDownloadExceptionTest.php | 61 +++++++
tests/Multipart/DownloadStateTest.php | 169 +-----------------
tests/Multipart/TestDownloader.php | 88 +++++++++
.../S3MultipartDownloadExceptionTest.php | 10 +-
tests/S3/MultipartDownloaderTest.php | 6 +-
10 files changed, 327 insertions(+), 364 deletions(-)
create mode 100644 tests/Exception/MultipartDownloadExceptionTest.php
create mode 100644 tests/Multipart/TestDownloader.php
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
index 1787ca2d69..48bbcc4924 100644
--- a/src/Multipart/AbstractDownloadManager.php
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -14,7 +14,7 @@
use Psr\Http\Message\RequestInterface;
/**
- * Encapsulates the execution of a multipart upload to S3 or Glacier.
+ * Encapsulates the execution of a multipart download to S3.
*
* @internal
*/
@@ -29,24 +29,24 @@ abstract class AbstractDownloadManager implements Promise\PromisorInterface
'concurrency' => self::DEFAULT_CONCURRENCY,
'prepare_data_source' => null,
'before_initiate' => null,
- 'before_upload' => null,
+ 'before_download' => null,
'before_complete' => null,
'exception_class' => 'Aws\Exception\MultipartDownloadException',
];
- /** @var Client Client used for the upload. */
+ /** @var Client Client used for the download. */
protected $client;
- /** @var array Configuration used to perform the upload. */
+ /** @var array Configuration used to perform the download. */
protected $config;
- /** @var array Service-specific information about the upload workflow. */
+ /** @var array Service-specific information about the download workflow. */
protected $info;
- /** @var PromiseInterface Promise that represents the multipart upload. */
+ /** @var PromiseInterface Promise that represents the multipart download. */
protected $promise;
- /** @var UploadState State used to manage the upload. */
+ /** @var DownloadState State used to manage the download. */
protected $state;
/**
@@ -56,15 +56,15 @@ abstract class AbstractDownloadManager implements Promise\PromisorInterface
public function __construct(Client $client, array $config = [])
{
$this->client = $client;
- $this->info = $this->loadUploadWorkflowInfo();
+ $this->info = $this->loadDownloadWorkflowInfo();
$this->config = $config + self::$defaultConfig;
$this->state = $this->determineState();
}
/**
- * Returns the current state of the upload
+ * Returns the current state of the download.
*
- * @return UploadState
+ * @return DownloadState
*/
public function getState()
{
@@ -72,11 +72,11 @@ public function getState()
}
/**
- * Upload the source using multipart upload operations.
+ * Download the source using multipart download operations.
*
- * @return Result The result of the CompleteMultipartUpload operation.
- * @throws \LogicException if the upload is already complete or aborted.
- * @throws MultipartUploadException if an upload operation fails.
+ * @return Result The result of the GetObject operations.
+ * @throws \LogicException if the download is already complete or aborted.
+ * @throws MultipartDownloadException if a download operation fails.
*/
public function download()
{
@@ -84,7 +84,7 @@ public function download()
}
/**
- * Upload the source asynchronously using multipart upload operations.
+ * Download the source asynchronously using multipart download operations.
*
* @return PromiseInterface
*/
@@ -95,9 +95,9 @@ public function promise()
}
return $this->promise = Promise\Coroutine::of(function () {
- // Initiate the upload.
+ // Initiate the download.
if ($this->state->isCompleted()) {
- throw new \LogicException('This multipart upload has already '
+ throw new \LogicException('This multipart download has already '
. 'been completed or aborted.'
);
}
@@ -107,12 +107,13 @@ public function promise()
if (is_callable($this->config["prepare_data_source"])) {
$this->config["prepare_data_source"]();
}
- $type = $this->getUploadType();
+ $type = $this->getDownloadType();
$result = (yield $this->execCommand('initiate', $this->getInitiateParams($type)));
$this->determineSourceSize($result['ContentRange']);
- $this->state->setUploadId(
- $this->info['id']['upload_id'],
- $result[$this->info['id']['upload_id']]
+ $this->setStreamPositionArray();
+ $this->state->setDownloadId(
+ $this->info['id']['download_id'],
+ $result[$this->info['id']['download_id']]
);
$this->state->setStatus(DownloadState::INITIATED);
if (isset($type['multipart'])){
@@ -127,15 +128,15 @@ public function promise()
or $result['PartsCount']==1){
$this->state->setStatus(DownloadState::COMPLETED);
} else {
- // Create a command pool from a generator that yields UploadPart
- // commands for each upload part.
+ // Create a command pool from a generator that yields DownloadPart
+ // commands for each download part.
$resultHandler = $this->getResultHandler($errors);
$commands = new CommandPool(
$this->client,
- $this->getUploadCommands($resultHandler),
+ $this->getDownloadCommands($resultHandler),
[
'concurrency' => $this->config['concurrency'],
- 'before' => $this->config['before_upload'],
+ 'before' => $this->config['before_download'],
]
);
@@ -145,8 +146,7 @@ public function promise()
throw new $this->config['exception_class']($this->state, $errors);
}
- // Complete the multipart upload.
-// yield $this->execCommand('complete', $this->getCompleteParams());
+ // Complete the multipart download.
$this->state->setStatus(DownloadState::COMPLETED);
}})->otherwise($this->buildFailureCatch());
}
@@ -179,19 +179,19 @@ protected function getConfig()
}
/**
- * Provides service-specific information about the multipart upload
+ * Provides service-specific information about the multipart download
* workflow.
*
* This array of data should include the keys: 'command', 'id', and 'part_num'.
*
* @return array
*/
- abstract protected function loadUploadWorkflowInfo();
+ abstract protected function loadDownloadWorkflowInfo();
abstract protected function determineSourceSize($size);
/**
- * Determines the part size to use for upload parts.
+ * Determines the part size to use for download parts.
*
* Examines the provided partSize value and the source to determine the
* best possible part size.
@@ -204,7 +204,8 @@ abstract protected function determinePartSize();
/**
* Uses information from the Command and Result to determine which part was
- * uploaded and mark it as uploaded in the upload's state.
+ * downloaded and mark it as downloaded in the download's state, and sends
+ * information to be written to the destination stream.
*
* @param CommandInterface $command
* @param ResultInterface $result
@@ -215,26 +216,27 @@ abstract protected function handleResult(
);
/**
- * Gets the service-specific parameters used to initiate the upload.
+ * Gets the service-specific parameters used to initiate the download.
+ *
+ * @param array $configType Service-specific params for the operation.
*
* @return array
*/
- abstract protected function getInitiateParams($type);
-
- abstract protected function getUploadType();
+ abstract protected function getInitiateParams($configType);
/**
- * Gets the service-specific parameters used to complete the upload.
+ * Gets the service-specific parameters used to complete the download
+ * from the config.
*
* @return array
*/
- abstract protected function getCompleteParams();
+ abstract protected function getDownloadType();
/**
* Based on the config and service-specific workflow info, creates a
- * `Promise` for an `UploadState` object.
+ * `Promise` for a `DownloadState` object.
*
- * @return PromiseInterface A `Promise` that resolves to an `UploadState`.
+ * @return PromiseInterface A `Promise` that resolves to a `DownloadState`.
*/
private function determineState()
{
@@ -245,12 +247,12 @@ private function determineState()
// Otherwise, construct a new state from the provided identifiers.
$required = $this->info['id'];
- $id = [$required['upload_id'] => null];
- unset($required['upload_id']);
+ $id = [$required['download_id'] => null];
+ unset($required['download_id']);
foreach ($required as $key => $param) {
if (!$this->config[$key]) {
throw new IAE('You must provide a value for "' . $key . '" in '
- . 'your config for the MultipartUploader for '
+ . 'your config for the MultipartDownloader for '
. $this->client->getApi()->getServiceFullName() . '.');
}
$id[$param] = $this->config[$key];
@@ -262,7 +264,7 @@ private function determineState()
}
/**
- * Executes a MUP command with all of the parameters for the operation.
+ * Executes a MUP command with all the parameters for the operation.
*
* @param string $operation Name of the operation.
* @param array $params Service-specific params for the operation.
@@ -287,7 +289,7 @@ protected function execCommand($operation, array $params)
}
/**
- * Returns a middleware for processing responses of part upload operations.
+ * Returns a middleware for processing responses of part download operations.
*
* - Adds an onFulfilled callback that calls the service-specific
* handleResult method on the Result of the operation.
@@ -295,7 +297,7 @@ protected function execCommand($operation, array $params)
* - Has a passedByRef $errors arg that the exceptions get added to. The
* caller should use that &$errors array to do error handling.
*
- * @param array $errors Errors from upload operations are added to this.
+ * @param array $errors Errors from download operations are added to this.
*
* @return callable
*/
@@ -321,16 +323,15 @@ function (AwsException $e) use (&$errors) {
}
/**
- * Creates a generator that yields part data for the upload's source.
+ * Creates a generator that yields part data for the download's source.
*
* Yields associative arrays of parameters that are ultimately merged in
* with others to form the complete parameters of a command. This can
- * include the Body parameter, which is a limited stream (i.e., a Stream
- * object, decorated with a LimitStream).
+ * include the PartNumber or Range parameter.
*
* @param callable $resultHandler
*
* @return \Generator
*/
- abstract protected function getUploadCommands(callable $resultHandler);
+ abstract protected function getDownloadCommands(callable $resultHandler);
}
diff --git a/src/Multipart/AbstractDownloader.php b/src/Multipart/AbstractDownloader.php
index 03eb98e83c..9b74a2af7d 100644
--- a/src/Multipart/AbstractDownloader.php
+++ b/src/Multipart/AbstractDownloader.php
@@ -9,35 +9,38 @@
abstract class AbstractDownloader extends AbstractDownloadManager
{
- /** @var Stream Source of the data to be uploaded. */
+ /** @var Stream Source of the data to be downloaded. */
protected $source;
- protected $position = 0;
+ /** @var Numeric Current position to track beginning of part. */
+ protected $partPosition = 0;
+
+ /** @var int Size of source. */
+ protected $sourceSize;
/**
* @param Client $client
- * @param mixed $source
+ * @param mixed $dest
* @param array $config
*/
- public function __construct(Client $client, $source, array $config = [])
+ public function __construct(Client $client, $dest, array $config = [])
{
-// $this->source = $this->determineSource($source);
$this->config = $config;
parent::__construct($client, $config);
}
- protected function getUploadCommands(callable $resultHandler)
+ protected function getDownloadCommands(callable $resultHandler)
{
// Determine if the source can be seeked.
- for ($partNumber = 1; $this->isEof($this->position); $partNumber++) {
- // If we haven't already uploaded this part, yield a new part.
- if (!$this->state->hasPartBeenUploaded($partNumber)) {
- $partStartPos = $this->position;
- if (!($data = $this->createPart($partStartPos, $partNumber))) {
+ for ($partNumber = 1; $this->isEof($this->partPosition); $partNumber++) {
+ // If we haven't already downloaded this part, yield a new part.
+ if (!$this->state->hasPartBeenDownloaded($partNumber)) {
+ $partStartPosition = $this->partPosition;
+ if (!($data = $this->createPart($partStartPosition, $partNumber))) {
break;
}
$command = $this->client->getCommand(
- $this->info['command']['upload'],
+ $this->info['command']['download'],
$data + $this->state->getId()
);
$command->getHandlerList()->appendSign($resultHandler, 'mup');
@@ -52,30 +55,28 @@ protected function getUploadCommands(callable $resultHandler)
)
);
}
-
yield $command;
}
-
// Advance the source's offset if not already advanced.
- $this->position += $this->state->getPartSize();
+ $this->partPosition += $this->state->getPartSize();
}
}
/**
- * Generates the parameters for an upload part by analyzing a range of the
+ * Generates the parameters for a download part by analyzing a range of the
* source starting from the current offset up to the part size.
*
- * @param bool $seekable
- * @param int $number
+ * @param numeric $partStartPosition
+ * @param int $partNumber
*
* @return array|null
*/
- abstract protected function createPart($seekable, $number);
+ abstract protected function createPart($partStartPosition, $partNumber);
/**
* Checks if the source is at EOF.
*
- * @param bool $seekable
+ * @param numeric $position
*
* @return bool
*/
@@ -85,22 +86,24 @@ private function isEof($position)
}
/**
- * Turns the provided source into a stream and stores it.
- *
- * If a string is provided, it is assumed to be a filename, otherwise, it
- * passes the value directly to `Psr7\Utils::streamFor()`.
+ * Determines and sets size of the source.
*
- * @param mixed $source
+ * @param mixed $range
*
- * @return Stream
*/
protected function determineSourceSize($range)
{
$size = substr($range, strpos($range, "/") + 1);
$this->sourceSize = $size;
- $this->setStreamPositionArray($this->sourceSize);
}
+ /**
+ * Determines and sets number of parts.
+ *
+ * @param numeric $partSize
+ *
+ * @return float|null
+ */
protected function getNumberOfParts($partSize)
{
if ($this->sourceSize) {
diff --git a/src/Multipart/DownloadState.php b/src/Multipart/DownloadState.php
index aaa9782985..dfee662584 100644
--- a/src/Multipart/DownloadState.php
+++ b/src/Multipart/DownloadState.php
@@ -2,10 +2,10 @@
namespace Aws\Multipart;
/**
- * Representation of the multipart upload.
+ * Representation of the multipart download.
*
- * This object keeps track of the state of the upload, including the status and
- * which parts have been uploaded.
+ * This object keeps track of the state of the download, including the status and
+ * which parts have been downloaded.
*/
class DownloadState
{
@@ -13,20 +13,20 @@ class DownloadState
const INITIATED = 1;
const COMPLETED = 2;
- /** @var array Params used to identity the upload. */
+ /** @var array Params used to identity the download. */
private $id;
- /** @var int Part size being used by the upload. */
+ /** @var int Part size being used by the download. */
private $partSize;
- /** @var array Parts that have been uploaded. */
+ /** @var array Parts that have been downloaded. */
private $downloadedParts = [];
- /** @var int Identifies the status the upload. */
+ /** @var int Identifies the status the download. */
private $status = self::CREATED;
/**
- * @param array $id Params used to identity the upload.
+ * @param array $id Params used to identity the download.
*/
public function __construct(array $id)
{
@@ -34,8 +34,8 @@ public function __construct(array $id)
}
/**
- * Get the upload's ID, which is a tuple of parameters that can uniquely
- * identify the upload.
+ * Get the download's ID, which is a tuple of parameters that can uniquely
+ * identify the download.
*
* @return array
*/
@@ -45,15 +45,14 @@ public function getId()
}
/**
- * Set's the "upload_id", or 3rd part of the upload's ID. This typically
- * only needs to be done after initiating an upload.
+ * Set's the "download_id", or 3rd part of the download's ID. This typically
+ * only needs to be done after initiating a download.
*
- * @param string $key The param key of the upload_id.
- * @param string $value The param value of the upload_id.
+ * @param string $key The param key of the download_id.
+ * @param string $value The param value of the download_id.
*/
- public function setUploadId($key, $value)
+ public function setDownloadId($key, $value)
{
- // i don't think i need this, instead i need to be sending the size to here?
$this->id[$key] = $value;
}
@@ -70,7 +69,7 @@ public function getPartSize()
/**
* Set the part size.
*
- * @param $partSize int Size of upload parts.
+ * @param $partSize int Size of download parts.
*/
public function setPartSize($partSize)
{
@@ -78,10 +77,10 @@ public function setPartSize($partSize)
}
/**
- * Marks a part as being uploaded.
+ * Marks a part as being downloaded.
*
* @param string $partNumber The part number.
- * @param array $partData Data from the upload operation that needs to be
+ * @param array $partData Data from the download operation that needs to be
* recalled during the complete operation.
*/
public function markPartAsDownloaded($partNumber, array $partData = [])
@@ -90,19 +89,19 @@ public function markPartAsDownloaded($partNumber, array $partData = [])
}
/**
- * Returns whether a part has been uploaded.
+ * Returns whether a part has been downloaded.
*
* @param int $partNumber The part number.
*
* @return bool
*/
- public function hasPartBeenUploaded($partNumber)
+ public function hasPartBeenDownloaded($partNumber)
{
return isset($this->downloadedParts[$partNumber]);
}
/**
- * Returns a sorted list of all the uploaded parts.
+ * Returns a sorted list of all the downloaded parts.
*
* @return array
*/
@@ -113,7 +112,7 @@ public function getDownloadedParts()
}
/**
- * Set the status of the upload.
+ * Set the status of the download.
*
* @param int $status Status is an integer code defined by the constants
* CREATED, INITIATED, and COMPLETED on this class.
@@ -125,7 +124,7 @@ public function setStatus($status)
}
/**
- * Determines whether the upload state is in the INITIATED status.
+ * Determines whether the download state is in the INITIATED status.
*
* @return bool
*/
@@ -135,7 +134,7 @@ public function isInitiated()
}
/**
- * Determines whether the upload state is in the COMPLETED status.
+ * Determines whether the download state is in the COMPLETED status.
*
* @return bool
*/
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
index 89d7439279..8f2d26ec70 100644
--- a/src/S3/MultipartDownloader.php
+++ b/src/S3/MultipartDownloader.php
@@ -18,7 +18,6 @@ class MultipartDownloader extends AbstractDownloader
const PART_MIN_SIZE = 5242880;
const PART_MAX_SIZE = 5368709120;
- const PART_MAX_NUM = 10000;
public $destStream;
public $streamPositionArray;
@@ -28,40 +27,40 @@ class MultipartDownloader extends AbstractDownloader
*
* The valid configuration options are as follows:
*
- * - acl: (string) ACL to set on the object being download. Objects are
+ * - acl: (string) ACL to set on the object being downloaded. Objects are
* private by default.
* - before_complete: (callable) Callback to invoke before the
- * `CompleteMultipartUpload` operation. The callback should have a
+ * `GetObject` operation. The callback should have a
* function signature like `function (Aws\Command $command) {...}`.
* - before_initiate: (callable) Callback to invoke before the
- * `CreateMultipartUpload` operation. The callback should have a function
+ * `GetObject` operation. The callback should have a function
* signature like `function (Aws\Command $command) {...}`.
- * - before_upload: (callable) Callback to invoke before any `UploadPart`
+ * - before_download: (callable) Callback to invoke before any `DownloadPart`
* operations. The callback should have a function signature like
* `function (Aws\Command $command) {...}`.
* - bucket: (string, required) Name of the bucket to which the object is
- * being uploaded, or an S3 access point ARN.
+ * being downloaded, or an S3 access point ARN.
* - concurrency: (int, default=int(5)) Maximum number of concurrent
- * `UploadPart` operations allowed during the multipart upload.
- * - key: (string, required) Key to use for the object being uploaded.
+ * `DownloadPart` operations allowed during the multipart download.
+ * - key: (string, required) Key to use for the object being download.
* - params: (array) An array of key/value parameters that will be applied
- * to each of the sub-commands run by the uploader as a base.
+ * to each of the sub-commands run by the downloader as a base.
* Auto-calculated options will override these parameters. If you need
* more granularity over parameters to each sub-command, use the before_*
* options detailed above to update the commands directly.
* - part_size: (int, default=int(5242880)) Part size, in bytes, to use when
- * doing a multipart upload. This must between 5 MB and 5 GB, inclusive.
+ * doing a multipart download. This must between 5 MB and 5 GB, inclusive.
* - prepare_data_source: (callable) Callback to invoke before starting the
- * multipart upload workflow. The callback should have a function
+ * multipart downloaded workflow. The callback should have a function
* signature like `function () {...}`.
- * - state: (Aws\Multipart\UploadState) An object that represents the state
- * of the multipart upload and that is used to resume a previous upload.
+ * - state: (Aws\Multipart\DownloadState) An object that represents the state
+ * of the multipart download and that is used to resume a previous download.
* When this option is provided, the `bucket`, `key`, and `part_size`
* options are ignored.
*
- * @param S3ClientInterface $client Client used for the upload.
- * @param mixed $dest Destination for data to download.
- * @param array $config Configuration used to perform the upload.
+ * @param S3ClientInterface $client Client used for the download.
+ * @param string $dest Destination for data to download.
+ * @param array $config Configuration used to perform the download.
*/
public function __construct(
S3ClientInterface $client,
@@ -76,31 +75,30 @@ public function __construct(
]);
}
- protected function loadUploadWorkflowInfo()
+ protected function loadDownloadWorkflowInfo()
{
return [
'command' => [
'initiate' => 'GetObject',
- 'upload' => 'GetObject',
-// 'complete' => 'CompleteMultipartUpload',
+ 'download' => 'GetObject'
],
'id' => [
'bucket' => 'Bucket',
'key' => 'Key',
- 'upload_id' => 'UploadId',
+ 'download_id' => 'DownloadId',
],
'part_num' => 'PartNumber',
];
}
- protected function createPart($partStartPos, $number)
+ protected function createPart($partStartPosition, $number)
{
// Initialize the array of part data that will be returned.
$data = [];
- // Apply custom params to UploadPart data
+ // Apply custom params to DownloadPart data
$config = $this->getConfig();
- $params = isset($config['params']) ? $config['params'] : [];
+ $params = $config['params'] ?? [];
foreach ($params as $k => $v) {
$data[$k] = $v;
}
@@ -109,8 +107,8 @@ protected function createPart($partStartPos, $number)
if (isset($this->config['range']) or
isset($this->config['multipartdownloadtype'])
&& $this->config['multipartdownloadtype'] == 'Range'){
- $partEndPos = $partStartPos+self::PART_MIN_SIZE;
- $data['Range'] = 'bytes='.$partStartPos.'-'.$partEndPos;
+ $partEndPosition = $partStartPosition+$this->state->getPartSize();
+ $data['Range'] = 'bytes='.$partStartPosition.'-'.$partEndPosition;
} else {
$data['PartNumber'] = $number;
}
@@ -129,9 +127,13 @@ protected function extractETag(ResultInterface $result)
return $result['ETag'];
}
- public function setStreamPositionArray($sourceSize)
+ /**
+ * Sets streamPositionArray with information on beginning of each part,
+ * depending on config.
+ */
+ public function setStreamPositionArray()
{
- $parts = ceil($sourceSize/$this->state->getPartSize());
+ $parts = ceil($this->sourceSize/$this->state->getPartSize());
$position = 0;
if (isset($this->config['range']) or
(isset($this->config['multipartdownloadtype']) &&
@@ -148,6 +150,13 @@ public function setStreamPositionArray($sourceSize)
}
}
+ /**
+ * Turns the provided destination into a writable stream and stores it.
+ *
+ * @param string $filePath Destination to turn into stream.
+ *
+ * @return Psr7\LazyOpenStream
+ */
protected function createDestStream($filePath)
{
return new Psr7\LazyOpenStream($filePath, 'w');
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
index 7235299576..60df1a1315 100644
--- a/src/S3/MultipartDownloadingTrait.php
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -9,13 +9,13 @@
trait MultipartDownloadingTrait
{
/**
- * Creates an UploadState object for a multipart upload by querying the
- * service for the specified upload's information.
+ * Creates a DownloadState object for a multipart download by querying the
+ * service for the specified download's information.
*
- * @param S3ClientInterface $client S3Client used for the upload.
- * @param string $bucket Bucket for the multipart upload.
- * @param string $key Object key for the multipart upload.
- * @param string $uploadId Upload ID for the multipart upload.
+ * @param S3ClientInterface $client S3Client used for the download.
+ * @param string $bucket Bucket for the multipart download.
+ * @param string $key Object key for the multipart download.
+ * @param string $downloadId Download ID for the multipart download.
*
* @return DownloadState
*/
@@ -23,12 +23,12 @@ public static function getStateFromService(
S3ClientInterface $client,
$bucket,
$key,
- $uploadId
+ $downloadId
) {
$state = new DownloadState([
'Bucket' => $bucket,
'Key' => $key,
- 'UploadId' => $uploadId,
+ 'DownloadId' => $downloadId,
]);
foreach ($client->getPaginator('ListParts', $state->getId()) as $result) {
@@ -36,7 +36,7 @@ public static function getStateFromService(
if (!$state->getPartSize()) {
$state->setPartSize($result->search('Parts[0].Size'));
}
- // Mark all the parts returned by ListParts as uploaded.
+ // Mark all the parts returned by ListParts as downloaded.
foreach ($result['Parts'] as $part) {
$state->markPartAsDownloaded($part['PartNumber'], [
'PartNumber' => $part['PartNumber'],
@@ -46,36 +46,32 @@ public static function getStateFromService(
}
$state->setStatus(DownloadState::INITIATED);
-
return $state;
}
protected function handleResult($command, ResultInterface $result)
{
if (!($command instanceof CommandInterface)){
- // single downloads - part/range
- $this->getState()->markPartAsDownloaded(1, [
- 'PartNumber' => 1,
- 'ETag' => $this->extractETag($result),
- ]);
- $this->writeDestStream(0, $result['Body']);
+ // single part downloads - part and range
+ $partNumber = 1;
+ $position = 0;
} elseif (!(isset($command['PartNumber']))) {
- // multi downloads - range
+ // multipart downloads - range
$seek = substr($command['Range'], strpos($command['Range'], "=") + 1);
$seek = (int)(strtok($seek, '-'));
- $this->getState()->markPartAsDownloaded($this->streamPositionArray[$seek], [
- 'PartNumber' => $this->streamPositionArray[$seek],
- 'ETag' => $this->extractETag($result),
- ]);
- $this->writeDestStream($seek, $result['Body']);
+ $partNumber = $this->streamPositionArray[$seek];
+ $position = $seek;
} else {
- // multi downloads - part
- $this->getState()->markPartAsDownloaded($command['PartNumber'], [
- 'PartNumber' => $command['PartNumber'],
- 'ETag' => $this->extractETag($result),
- ]);
- $this->writeDestStream($this->streamPositionArray[$command['PartNumber']], $result['Body']);
+ // multipart downloads - part
+ $partNumber = $command['PartNumber'];
+ $position = $this->streamPositionArray[$command['PartNumber']];
}
+
+ $this->getState()->markPartAsDownloaded($partNumber, [
+ 'PartNumber' => $partNumber,
+ 'ETag' => $this->extractETag($result),
+ ]);
+ $this->writeDestStream($position, $result['Body']);
}
protected function writeDestStream($position, $body)
@@ -86,31 +82,11 @@ protected function writeDestStream($position, $body)
abstract protected function extractETag(ResultInterface $result);
- protected function getCompleteParams()
- {
- $config = $this->getConfig();
- $params = isset($config['params']) ? $config['params'] : [];
-
- $params['MultipartUpload'] = [
- 'Parts' => $this->getState()->getDownloadedParts()
- ];
-
- return $params;
- }
-
protected function determinePartSize()
{
// Make sure the part size is set.
$partSize = $this->getConfig()['part_size'] ?: MultipartDownloader::PART_MIN_SIZE;
- // Adjust the part size to be larger for known, x-large uploads.
-// if ($sourceSize = $this->getSourceSize()) {
-// $partSize = (int) max(
-// $partSize,
-// ceil($sourceSize / MultipartDownloader::PART_MAX_NUM)
-// );
-// }
-
// Ensure that the part size follows the rules: 5 MB <= size <= 5 GB.
if ($partSize < MultipartDownloader::PART_MIN_SIZE || $partSize > MultipartDownloader::PART_MAX_SIZE) {
throw new \InvalidArgumentException('The part size must be no less '
@@ -123,7 +99,7 @@ protected function determinePartSize()
protected function getInitiateParams($configType)
{
$config = $this->getConfig();
- $params = isset($config['params']) ? $config['params'] : [];
+ $params = $config['params'] ?? [];
if (isset($config['acl'])) {
$params['ACL'] = $config['acl'];
@@ -134,7 +110,7 @@ protected function getInitiateParams($configType)
return $params;
}
- protected function getUploadType()
+ protected function getDownloadType()
{
$config = $this->getConfig();
if (isset($config['partnumber'])) {
@@ -155,19 +131,8 @@ protected function getUploadType()
}
}
-// public function setStreamPosArray($sourceSize)
-// {
-// $parts = ceil($sourceSize/$this->partSize);
-// $position = 0;
-// for ($i=1;$i<=$parts;$i++) {
-// $this->StreamPosArray []= $position;
-// $position += $this->partSize;
-// }
-// print_r($this->streamPositionArray);
-// }
-
/**
- * @return UploadState
+ * @return DownloadState
*/
abstract protected function getState();
diff --git a/tests/Exception/MultipartDownloadExceptionTest.php b/tests/Exception/MultipartDownloadExceptionTest.php
new file mode 100644
index 0000000000..899cef8df0
--- /dev/null
+++ b/tests/Exception/MultipartDownloadExceptionTest.php
@@ -0,0 +1,61 @@
+assertSame(
+ "An exception occurred while {$status} a multipart upload: $msg",
+ $exception->getMessage()
+ );
+ $this->assertSame($state, $exception->getState());
+ $this->assertSame($prev, $exception->getPrevious());
+ }
+
+ public function getTestCases()
+ {
+ return [
+ ['GetObject', 'performing']
+ ];
+ }
+
+ public function testCanCreateExceptionListingFailedParts()
+ {
+ $state = new DownloadState([]);
+ $failed = [
+ 1 => new AwsException('Bad digest.', new Command('GetObject')),
+ 5 => new AwsException('Missing header.', new Command('GetObject')),
+ 8 => new AwsException('Needs more love.', new Command('GetObject')),
+ ];
+
+ $exception = new MultipartDownloadException($state, $failed);
+
+ $expected = <<assertSame($expected, $exception->getMessage());
+ }
+}
diff --git a/tests/Multipart/DownloadStateTest.php b/tests/Multipart/DownloadStateTest.php
index 3ccfaefaae..8bf9ef2fd2 100644
--- a/tests/Multipart/DownloadStateTest.php
+++ b/tests/Multipart/DownloadStateTest.php
@@ -43,12 +43,12 @@ public function testCanTrackDownloadedParts()
$state = new DownloadState([]);
$this->assertEmpty($state->getDownloadedParts());
- $state->markPartAsUploaded(1, ['foo' => 1]);
- $state->markPartAsUploaded(3, ['foo' => 3]);
- $state->markPartAsUploaded(2, ['foo' => 2]);
+ $state->markPartAsDownloaded(1, ['foo' => 1]);
+ $state->markPartAsDownloaded(3, ['foo' => 3]);
+ $state->markPartAsDownloaded(2, ['foo' => 2]);
- $this->assertTrue($state->hasPartBeenUploaded(2));
- $this->assertFalse($state->hasPartBeenUploaded(5));
+ $this->assertTrue($state->hasPartBeenDownloaded(2));
+ $this->assertFalse($state->hasPartBeenDownloaded(5));
// Note: The parts should come out sorted.
$this->assertSame([1, 2, 3], array_keys($state->getDownloadedParts()));
@@ -70,163 +70,4 @@ public function testSerializationWorks()
$this->assertTrue($newState->isInitiated());
$this->assertArrayHasKey('foo', $newState->getId());
}
-
-// public function testEmptyUploadStateOutputWithConfigFalse()
-// {
-// $config['track_upload'] = false;
-// $state = new DownloadState([], $config);
-// $state->setProgressThresholds(100);
-// $state->displayProgress(13);
-// $this->expectOutputString('');
-// }
-
- /**
- * @dataProvider getDisplayProgressCases
- */
-// public function testDisplayProgressPrintsProgress(
-// $totalSize,
-// $totalUploaded,
-// $progressBar
-// ) {
-// $config['track_upload'] = true;
-// $state = new UploadState([]);
-// $state->setProgressThresholds($totalSize);
-// $state->displayProgress($totalUploaded);
-//
-// $this->expectOutputString($progressBar);
-// }
-
-// public function getDisplayProgressCases()
-// {
-// $progressBar = ["Transfer initiated...\n| | 0.0%\n",
-// "|== | 12.5%\n",
-// "|===== | 25.0%\n",
-// "|======= | 37.5%\n",
-// "|========== | 50.0%\n",
-// "|============ | 62.5%\n",
-// "|=============== | 75.0%\n",
-// "|================= | 87.5%\n",
-// "|====================| 100.0%\nTransfer complete!\n"];
-// return [
-// [100000, 0, $progressBar[0]],
-// [100000, 12499, $progressBar[0]],
-// [100000, 12500, "{$progressBar[0]}{$progressBar[1]}"],
-// [100000, 24999, "{$progressBar[0]}{$progressBar[1]}"],
-// [100000, 25000, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"],
-// [100000, 37499, "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}"],
-// [
-// 100000,
-// 37500,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}"
-// ],
-// [
-// 100000,
-// 49999,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}"
-// ],
-// [
-// 100000,
-// 50000,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}"
-// ],
-// [
-// 100000,
-// 62499,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}"
-// ],
-// [
-// 100000,
-// 62500,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
-// "{$progressBar[5]}"
-// ],
-// [
-// 100000,
-// 74999,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
-// "{$progressBar[5]}"
-// ],
-// [
-// 100000,
-// 75000,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
-// "{$progressBar[5]}{$progressBar[6]}"
-// ],
-// [
-// 100000,
-// 87499,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
-// "{$progressBar[5]}{$progressBar[6]}"
-// ],
-// [
-// 100000,
-// 87500,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
-// "{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}"
-// ],
-// [
-// 100000,
-// 99999,
-// "{$progressBar[0]}{$progressBar[1]}{$progressBar[2]}{$progressBar[3]}{$progressBar[4]}" .
-// "{$progressBar[5]}{$progressBar[6]}{$progressBar[7]}"
-// ],
-// [100000, 100000, implode($progressBar)]
-// ];
-// }
-//
-// /**
-// * @dataProvider getThresholdCases
-// */
-// public function testUploadThresholds($totalSize)
-// {
-// $config['track_upload'] = true;
-// $state = new UploadState([]);
-// $threshold = $state->setProgressThresholds($totalSize);
-//
-// $this->assertIsArray($threshold);
-// $this->assertCount(9, $threshold);
-// }
-//
-// public function getThresholdCases()
-// {
-// return [
-// [0],
-// [100000],
-// [100001]
-// ];
-// }
-//
-// /**
-// * @dataProvider getInvalidIntCases
-// */
-// public function testSetProgressThresholdsThrowsException($totalSize)
-// {
-// $state = new UploadState([]);
-// $this->expectExceptionMessage('The total size of the upload must be an int.');
-// $this->expectException(\InvalidArgumentException::class);
-//
-// $state->setProgressThresholds($totalSize);
-// }
-//
-// /**
-// * @dataProvider getInvalidIntCases
-// */
-// public function testDisplayProgressThrowsException($totalUploaded)
-// {
-// $state = new UploadState([]);
-// $this->expectExceptionMessage('The size of the bytes being uploaded must be an int.');
-// $this->expectException(\InvalidArgumentException::class);
-//
-// $state->displayProgress($totalUploaded);
-// }
-//
-// public function getInvalidIntCases()
-// {
-// return [
-// [''],
-// [null],
-// ['1234'],
-// ['aws'],
-// ];
-// }
}
\ No newline at end of file
diff --git a/tests/Multipart/TestDownloader.php b/tests/Multipart/TestDownloader.php
new file mode 100644
index 0000000000..174e167ace
--- /dev/null
+++ b/tests/Multipart/TestDownloader.php
@@ -0,0 +1,88 @@
+destStream = $this->createDestStream($dest);
+ parent::__construct($client, $dest, $config + [
+ 'bucket' => null,
+ 'key' => null,
+ 'exception_class' => S3MultipartDownloadException::class,
+ ]);
+ }
+ protected function loadUploadWorkflowInfo()
+ {
+ return [
+ 'command' => [
+ 'initiate' => 'GetObject',
+ 'upload' => 'GetObject',
+ ],
+ 'id' => [
+ 'bucket' => 'Bucket',
+ 'key' => 'Key',
+ 'upload_id' => 'UploadId',
+ ],
+ 'part_num' => 'PartNumber',
+ ];
+ }
+
+ protected function determinePartSize()
+ {
+ return $this->config['part_size'] ?: 2;
+ }
+
+ protected function getInitiateParams($configType)
+ {
+ return [];
+ }
+
+ protected function createPart($seekable, $number)
+ {
+// if ($seekable) {
+// $body = Psr7\Utils::streamFor(fopen($this->source->getMetadata('uri'), 'r'));
+// $body = $this->limitPartStream($body);
+// } else {
+// $body = Psr7\Utils::streamFor($this->source->read($this->state->getPartSize()));
+// }
+
+ // Do not create a part if the body size is zero.
+// if ($body->getSize() === 0) {
+// return false;
+// }
+
+ return [
+ 'PartNumber' => $number,
+// 'Body' => $body,
+// 'UploadId' => 'baz'
+ ];
+ }
+
+ protected function handleResult(CommandInterface $command, ResultInterface $result)
+ {
+ $this->state->markPartAsUploaded($command['PartNumber'], [
+ 'PartNumber' => $command['PartNumber'],
+ 'ETag' => $result['ETag']
+ ]);
+ }
+
+ protected function getCompleteParams()
+ {
+ return [
+ 'MultipartUpload' => [
+ 'Parts' => $this->state->getUploadedParts()
+ ],
+ 'UploadId' => 'baz'
+ ];
+ }
+}
diff --git a/tests/S3/Exception/S3MultipartDownloadExceptionTest.php b/tests/S3/Exception/S3MultipartDownloadExceptionTest.php
index c7df556c52..d166b6ed08 100644
--- a/tests/S3/Exception/S3MultipartDownloadExceptionTest.php
+++ b/tests/S3/Exception/S3MultipartDownloadExceptionTest.php
@@ -20,22 +20,18 @@ public function testCanProviderFailedTransferFilePathInfo()
1 => new AwsException('Bad digest.', new Command('GetObject', [
'Bucket' => 'foo',
'Key' => 'bar'
-// 'Body' => Psr7\Utils::streamFor('Part 1'),
])),
5 => new AwsException('Missing header.', new Command('GetObject', [
'Bucket' => 'foo',
- 'Key' => 'bar',
-// 'Body' => Psr7\Utils::streamFor('Part 2'),
+ 'Key' => 'bar'
])),
8 => new AwsException('Needs more love.', new Command('GetObject')),
];
$path = '/path/to/the/large/file/test.zip';
- $exception = new S3MultipartDownloadException($state, $failed, [
- 'file_name' => $path
- ]);
+ $exception = new S3MultipartDownloadException($state, $failed);
$this->assertSame('foo', $exception->getBucket());
$this->assertSame('bar', $exception->getKey());
- $this->assertSame('php://temp', $exception->getSourceFileName());
+// $this->assertSame('php://temp', $exception->getSourceFileName());
}
}
diff --git a/tests/S3/MultipartDownloaderTest.php b/tests/S3/MultipartDownloaderTest.php
index 289e35b322..3ea82ae9a7 100644
--- a/tests/S3/MultipartDownloaderTest.php
+++ b/tests/S3/MultipartDownloaderTest.php
@@ -53,10 +53,10 @@ public function testS3MultipartDownloadWorkflow(
}
}
- $uploader = new MultipartDownloader($client, $dest, $uploadOptions);
- $result = $uploader->download();
+ $downloader = new MultipartDownloader($client, $dest, $uploadOptions);
+ $result = $downloader->download();
- $this->assertTrue($uploader->getState()->isCompleted());
+ $this->assertTrue($downloader->getState()->isCompleted());
$this->assertSame($url, $result['ObjectURL']);
}
From 13eff472efacf39cdb6fe00e68ffc735c2ea248d Mon Sep 17 00:00:00 2001
From: Cintia <81837982+cintiashamsu@users.noreply.github.com>
Date: Mon, 31 Jul 2023 11:28:47 -0400
Subject: [PATCH 31/31] downloader + checksum + tests unfinished
---
src/Exception/MultipartDownloadException.php | 10 +-
src/Multipart/AbstractDownloadManager.php | 14 +-
src/Multipart/DownloadState.php | 73 +++++-
src/S3/MultipartDownloader.php | 12 +-
src/S3/MultipartDownloadingTrait.php | 65 ++++--
tests/Multipart/AbstractDownloaderTest.php | 92 +++-----
tests/Multipart/DownloadStateTest.php | 11 +-
tests/Multipart/TestDownloader.php | 127 +++++++---
tests/S3/MultipartDownloaderTest.php | 233 ++++++++-----------
9 files changed, 372 insertions(+), 265 deletions(-)
diff --git a/src/Exception/MultipartDownloadException.php b/src/Exception/MultipartDownloadException.php
index ad35714f37..948ff493b7 100644
--- a/src/Exception/MultipartDownloadException.php
+++ b/src/Exception/MultipartDownloadException.php
@@ -18,10 +18,10 @@ class MultipartDownloadException extends \RuntimeException implements
* @param \Exception|array $prev Exception being thrown.
*/
public function __construct(DownloadState $state, $prev = null) {
- $msg = 'An exception occurred while performing a multipart upload';
+ $msg = 'An exception occurred while performing a multipart download';
if (is_array($prev)) {
- $msg = strtr($msg, ['performing' => 'uploading parts to']);
+ $msg = strtr($msg, ['performing' => 'downloading parts to']);
$msg .= ". The following parts had errors:\n";
/** @var $error AwsException */
foreach ($prev as $part => $error) {
@@ -29,11 +29,11 @@ public function __construct(DownloadState $state, $prev = null) {
}
} elseif ($prev instanceof AwsException) {
switch ($prev->getCommand()->getName()) {
- case 'CreateMultipartUpload':
- case 'InitiateMultipartUpload':
+ case 'GetObject':
+ case 'GetObject':
$action = 'initiating';
break;
- case 'CompleteMultipartUpload':
+ case 'GetObject':
$action = 'completing';
break;
}
diff --git a/src/Multipart/AbstractDownloadManager.php b/src/Multipart/AbstractDownloadManager.php
index 48bbcc4924..97a7b0c2ef 100644
--- a/src/Multipart/AbstractDownloadManager.php
+++ b/src/Multipart/AbstractDownloadManager.php
@@ -110,11 +110,10 @@ public function promise()
$type = $this->getDownloadType();
$result = (yield $this->execCommand('initiate', $this->getInitiateParams($type)));
$this->determineSourceSize($result['ContentRange']);
+ if ($this->getState()->displayProgress) {
+ $this->state->setProgressThresholds($this->sourceSize);
+ }
$this->setStreamPositionArray();
- $this->state->setDownloadId(
- $this->info['id']['download_id'],
- $result[$this->info['id']['download_id']]
- );
$this->state->setStatus(DownloadState::INITIATED);
if (isset($type['multipart'])){
$this->handleResult(1, $result);
@@ -127,6 +126,9 @@ public function promise()
or isset($this->config['range'])
or $result['PartsCount']==1){
$this->state->setStatus(DownloadState::COMPLETED);
+ if ($this->getState()->displayProgress) {
+ echo end($this->state->progressBar);
+ }
} else {
// Create a command pool from a generator that yields DownloadPart
// commands for each download part.
@@ -246,9 +248,9 @@ private function determineState()
}
// Otherwise, construct a new state from the provided identifiers.
+ // TODO delete id logic
$required = $this->info['id'];
- $id = [$required['download_id'] => null];
- unset($required['download_id']);
+ $id = [];
foreach ($required as $key => $param) {
if (!$this->config[$key]) {
throw new IAE('You must provide a value for "' . $key . '" in '
diff --git a/src/Multipart/DownloadState.php b/src/Multipart/DownloadState.php
index dfee662584..d422bdc073 100644
--- a/src/Multipart/DownloadState.php
+++ b/src/Multipart/DownloadState.php
@@ -13,11 +13,23 @@ class DownloadState
const INITIATED = 1;
const COMPLETED = 2;
+ public $progressBar = [
+ "Transfer initiated...\n| | 0.0%\n",
+ "|== | 12.5%\n",
+ "|===== | 25.0%\n",
+ "|======= | 37.5%\n",
+ "|========== | 50.0%\n",
+ "|============ | 62.5%\n",
+ "|=============== | 75.0%\n",
+ "|================= | 87.5%\n",
+ "|====================| 100.0%\nTransfer complete!\n"
+ ];
+
/** @var array Params used to identity the download. */
private $id;
/** @var int Part size being used by the download. */
- private $partSize;
+ public $partSize;
/** @var array Parts that have been downloaded. */
private $downloadedParts = [];
@@ -25,6 +37,12 @@ class DownloadState
/** @var int Identifies the status the download. */
private $status = self::CREATED;
+ /** @var array Thresholds for progress of the upload. */
+ private $progressThresholds = [];
+
+ /** @var boolean Determines status for tracking the upload */
+ public $displayProgress = false;
+
/**
* @param array $id Params used to identity the download.
*/
@@ -51,10 +69,10 @@ public function getId()
* @param string $key The param key of the download_id.
* @param string $value The param value of the download_id.
*/
- public function setDownloadId($key, $value)
- {
- $this->id[$key] = $value;
- }
+// public function setDownloadId($key, $value)
+// {
+// $this->id[$key] = $value;
+// }
/**
* Get the part size.
@@ -76,6 +94,51 @@ public function setPartSize($partSize)
$this->partSize = $partSize;
}
+ /**
+ * Sets the 1/8th thresholds array. $totalSize is only sent if
+ * 'track_download' is true.
+ *
+ * @param $totalSize numeric Size of object to download.
+ *
+ * @return array
+ */
+ public function setProgressThresholds($totalSize)
+ {
+ if(!is_numeric($totalSize)) {
+ throw new \InvalidArgumentException(
+ 'The total size of the upload must be a number.'
+ );
+ }
+
+ $this->progressThresholds[0] = 0;
+ for ($i=1;$i<=8;$i++) {
+ $this->progressThresholds []= round($totalSize*($i/8));
+ }
+ return $this->progressThresholds;
+ }
+
+ /**
+ * Prints progress of download.
+ *
+ * @param $totalUploaded numeric Size of download so far.
+ */
+ public function getDisplayProgress($totalUploaded)
+ {
+ if (!is_numeric($totalUploaded)) {
+ throw new \InvalidArgumentException(
+ 'The size of the bytes being uploaded must be a number.'
+ );
+ }
+
+ if ($this->displayProgress) {
+ while (!empty($this->progressBar)
+ && $totalUploaded >= $this->progressThresholds[0]) {
+ echo array_shift($this->progressBar);
+ array_shift($this->progressThresholds);
+ }
+ }
+ }
+
/**
* Marks a part as being downloaded.
*
diff --git a/src/S3/MultipartDownloader.php b/src/S3/MultipartDownloader.php
index 8f2d26ec70..02445a3711 100644
--- a/src/S3/MultipartDownloader.php
+++ b/src/S3/MultipartDownloader.php
@@ -22,6 +22,8 @@ class MultipartDownloader extends AbstractDownloader
public $destStream;
public $streamPositionArray;
+ protected $checksumMode = true;
+
/**
* Creates a multipart download for an S3 object.
*
@@ -73,6 +75,14 @@ public function __construct(
'key' => null,
'exception_class' => S3MultipartDownloadException::class,
]);
+
+ if (isset($config['checksum_validation_enabled'])
+ && !$config['checksum_validation_enabled']) {
+ $this->checksumMode = false;
+ }
+ if (isset($this->config['track_download']) && ($this->config['track_download'])) {
+ $this->getState()->displayProgress = true;
+ }
}
protected function loadDownloadWorkflowInfo()
@@ -85,7 +95,7 @@ protected function loadDownloadWorkflowInfo()
'id' => [
'bucket' => 'Bucket',
'key' => 'Key',
- 'download_id' => 'DownloadId',
+// 'download_id' => 'DownloadId',
],
'part_num' => 'PartNumber',
];
diff --git a/src/S3/MultipartDownloadingTrait.php b/src/S3/MultipartDownloadingTrait.php
index 60df1a1315..ceb0a3f4a9 100644
--- a/src/S3/MultipartDownloadingTrait.php
+++ b/src/S3/MultipartDownloadingTrait.php
@@ -8,6 +8,8 @@
trait MultipartDownloadingTrait
{
+ private $downloadedBytes = 0;
+
/**
* Creates a DownloadState object for a multipart download by querying the
* service for the specified download's information.
@@ -15,7 +17,6 @@ trait MultipartDownloadingTrait
* @param S3ClientInterface $client S3Client used for the download.
* @param string $bucket Bucket for the multipart download.
* @param string $key Object key for the multipart download.
- * @param string $downloadId Download ID for the multipart download.
*
* @return DownloadState
*/
@@ -23,26 +24,29 @@ public static function getStateFromService(
S3ClientInterface $client,
$bucket,
$key,
- $downloadId
+ $dest
) {
$state = new DownloadState([
+ 'Bucket' => $bucket,
+ 'Key' => $key
+ ]);
+
+ $info = $client->headObject([
'Bucket' => $bucket,
'Key' => $key,
- 'DownloadId' => $downloadId,
+// 'ChecksumMode' => 'ENABLED'
]);
- foreach ($client->getPaginator('ListParts', $state->getId()) as $result) {
- // Get the part size from the first part in the first result.
- if (!$state->getPartSize()) {
- $state->setPartSize($result->search('Parts[0].Size'));
- }
- // Mark all the parts returned by ListParts as downloaded.
- foreach ($result['Parts'] as $part) {
- $state->markPartAsDownloaded($part['PartNumber'], [
- 'PartNumber' => $part['PartNumber'],
- 'ETag' => $part['ETag']
- ]);
- }
+ $totalSize = $info['ContentLength'];
+ $state->setPartSize(1048576);
+ $partSize = $state->getPartSize();
+ $destStream = new Psr7\LazyOpenStream($dest, 'rw');
+
+ for ($byte = 0; $byte <= $totalSize; $byte+=$partSize) {
+ $stream = new Psr7\LimitStream($destStream, $partSize, $byte);
+ echo $stream->getSize() . "\n";
+ echo $stream->tell() . "\n";
+ // mark part as downloaded as you check
}
$state->setStatus(DownloadState::INITIATED);
@@ -51,6 +55,12 @@ public static function getStateFromService(
protected function handleResult($command, ResultInterface $result)
{
+ if ($this->checksumMode) {
+ if ($this->validateChecksum($result)){
+ throw new \Exception('Checksum invalid.');
+ }
+ }
+
if (!($command instanceof CommandInterface)){
// single part downloads - part and range
$partNumber = 1;
@@ -66,12 +76,31 @@ protected function handleResult($command, ResultInterface $result)
$partNumber = $command['PartNumber'];
$position = $this->streamPositionArray[$command['PartNumber']];
}
-
$this->getState()->markPartAsDownloaded($partNumber, [
'PartNumber' => $partNumber,
'ETag' => $this->extractETag($result),
]);
+
$this->writeDestStream($position, $result['Body']);
+ if ($this->getState()->displayProgress) {
+ $this->downloadedBytes+=strlen($result['Body']);
+ $this->getState()->getDisplayProgress($this->downloadedBytes);
+ }
+ }
+
+ private function validateChecksum($result)
+ {
+ if (isset($result['ChecksumValidated'])) {
+ $checksum = CalculatesChecksumTrait::getEncodedValue(
+ $result['ChecksumValidated'], $result['Body']
+ );
+ if ($checksum != $result['Checksum' . $result['ChecksumValidated']]) {
+ return true;
+ }
+ } elseif (!strpos($result['ETag'], '-')
+ && $result['ETag'] != md5($result['Body'])) {
+ return true;
+ }
}
protected function writeDestStream($position, $body)
@@ -107,6 +136,10 @@ protected function getInitiateParams($configType)
$params[$configType['config']] = $configType['configParam'];
+ if (isset($this->checksumMode)) {
+ $params['ChecksumMode'] = 'ENABLED';
+ }
+
return $params;
}
diff --git a/tests/Multipart/AbstractDownloaderTest.php b/tests/Multipart/AbstractDownloaderTest.php
index e1968343e9..ce058f3e60 100644
--- a/tests/Multipart/AbstractDownloaderTest.php
+++ b/tests/Multipart/AbstractDownloaderTest.php
@@ -4,6 +4,7 @@
use Aws\Command;
use Aws\Exception\AwsException;
use Aws\Exception\MultipartDownloadException;
+use Aws\S3\MultipartDownloader;
use Aws\Multipart\DownloadState;
use Aws\Result;
use Aws\Test\UsesServiceTrait;
@@ -24,14 +25,12 @@ private function getDownloaderWithState($status, array $results = [], $source =
$state->setStatus($status);
return $this->getTestDownloader(
- $source ?: Psr7\Utils::streamFor(),
['state' => $state],
$results
);
}
private function getTestDownloader(
- $source = null,
array $config = [],
array $results = []
) {
@@ -41,7 +40,7 @@ private function getTestDownloader(
]);
$this->addMockResults($client, $results);
- return new TestDownloader($client, $source ?: Psr7\Utils::streamFor(), $config);
+ return new MultipartDownloader($client, 'php://temp', $config);
}
public function testThrowsExceptionOnBadInitiateRequest()
@@ -55,39 +54,40 @@ public function testThrowsExceptionOnBadInitiateRequest()
public function testThrowsExceptionIfStateIsCompleted()
{
+ // set exception to expect
+ // set state as completed
+ // make sure state is completed
+ // check that exception is thrown
$this->expectException(\LogicException::class);
- $uploader = $this->getUploaderWithState(UploadState::COMPLETED);
- $this->assertTrue($uploader->getState()->isCompleted());
- $uploader->upload();
+ $downloader = $this->getDownloaderWithState(DownloadState::COMPLETED);
+ $this->assertTrue($downloader->getState()->isCompleted());
+ $downloader->download();
}
public function testSuccessfulCompleteReturnsResult()
{
- $uploader = $this->getUploaderWithState(UploadState::CREATED, [
- new Result(), // Initiate
- new Result(), // Upload
- new Result(), // Upload
- new Result(), // Upload
- new Result(['test' => 'foo']) // Complete
- ], Psr7\Utils::streamFor('abcdef'));
- $this->assertSame('foo', $uploader->upload()['test']);
- $this->assertTrue($uploader->getState()->isCompleted());
+ //
+ $downloader = $this->getDownloaderWithState(DownloadState::CREATED, [
+ new Result(['body' => Psr7\Utils::streamFor(str_repeat('.', 1 * 1048576))])
+ ], 'php://temp');
+ $this->assertSame(str_repeat('.', 1 * 1048576), $downloader->download()['body']);
+ $this->assertTrue($downloader->getState()->isCompleted());
}
public function testThrowsExceptionOnBadCompleteRequest()
{
- $this->expectException(\Aws\S3\Exception\S3MultipartUploadException::class);
- $uploader = $this->getUploaderWithState(UploadState::CREATED, [
+ $this->expectException(\Aws\S3\Exception\S3MultipartDownloadException::class);
+ $uploader = $this->getDownloaderWithState(DownloadState::CREATED, [
new Result(), // Initiate
new Result(), // Upload
new AwsException('Failed', new Command('Complete')),
- ], Psr7\Utils::streamFor('a'));
- $uploader->upload();
+ ], 'php://temp');
+ $uploader->download();
}
public function testThrowsExceptionOnBadUploadRequest()
{
- $uploader = $this->getUploaderWithState(UploadState::CREATED, [
+ $uploader = $this->getDownloaderWithState(DownloadState::CREATED, [
new Result(), // Initiate
new AwsException('Failed[1]', new Command('Upload', ['PartNumber' => 1])),
new Result(), // Upload
@@ -97,9 +97,9 @@ public function testThrowsExceptionOnBadUploadRequest()
], Psr7\Utils::streamFor('abcdefghi'));
try {
- $uploader->upload();
+ $uploader->download();
$this->fail('No exception was thrown.');
- } catch (MultipartUploadException $e) {
+ } catch (MultipartDownloadException $e) {
$message = $e->getMessage();
$this->assertStringContainsString('Failed[1]', $message);
$this->assertStringContainsString('Failed[4]', $message);
@@ -112,7 +112,7 @@ public function testThrowsExceptionOnBadUploadRequest()
// Test if can resume an upload.
$serializedState = serialize($e->getState());
$state = unserialize($serializedState);
- $secondChance = $this->getTestUploader(
+ $secondChance = $this->getTestDownloader(
Psr7\Utils::streamFor('abcdefghi'),
['state' => $state],
[
@@ -133,7 +133,7 @@ public function testAsyncUpload()
$called++;
};
- $uploader = $this->getTestUploader(Psr7\Utils::streamFor('abcde'), [
+ $uploader = $this->getTestDownloader(Psr7\Utils::streamFor('abcde'), [
'bucket' => 'foo',
'key' => 'bar',
'prepare_data_source' => $fn,
@@ -157,45 +157,19 @@ public function testAsyncUpload()
public function testRequiresIdParams()
{
$this->expectException(\InvalidArgumentException::class);
- $this->getTestUploader(Psr7\Utils::streamFor());
- }
-
- public function testCanSetSourceFromFilenameIfExists()
- {
- $config = ['bucket' => 'foo', 'key' => 'bar'];
-
- // CASE 1: Filename exists.
- $uploader = $this->getTestUploader(__FILE__, $config);
- $this->assertInstanceOf(
- 'Psr\Http\Message\StreamInterface',
- $this->getPropertyValue($uploader, 'source')
- );
-
- // CASE 2: Filename does not exist.
- $exception = null;
- try {
- $this->getTestUploader('non-existent-file.foobar', $config);
- } catch (\Exception $exception) {}
- $this->assertInstanceOf('RuntimeException', $exception);
-
- // CASE 3: Source stream is not readable.
- $exception = null;
- try {
- $this->getTestUploader(STDERR, $config);
- } catch (\Exception $exception) {}
- $this->assertInstanceOf('InvalidArgumentException', $exception);
+ $this->getTestDownloader();
}
/**
* @param bool $seekable
- * @param UploadState $state
+ * @param DownloadState $state
* @param array $expectedBodies
*
* @dataProvider getPartGeneratorTestCases
*/
public function testCommandGeneratorYieldsExpectedUploadCommands(
$seekable,
- UploadState $state,
+ DownloadState $state,
array $expectedBodies
) {
$source = Psr7\Utils::streamFor(fopen(__DIR__ . '/source.txt', 'r'));
@@ -203,7 +177,7 @@ public function testCommandGeneratorYieldsExpectedUploadCommands(
$source = new Psr7\NoSeekStream($source);
}
- $uploader = $this->getTestUploader($source, ['state' => $state]);
+ $uploader = $this->getTestDownloader($source, ['state' => $state]);
$uploader->getState();
$handler = function (callable $handler) {
return function ($c, $r) use ($handler) {
@@ -213,7 +187,7 @@ public function testCommandGeneratorYieldsExpectedUploadCommands(
$actualBodies = [];
$getUploadCommands = (new \ReflectionObject($uploader))
- ->getMethod('getUploadCommands');
+ ->getMethod('getDownloadCommands');
$getUploadCommands->setAccessible(true);
foreach ($getUploadCommands->invoke($uploader, $handler) as $cmd) {
$actualBodies[$cmd['PartNumber']] = $cmd['Body']->getContents();
@@ -234,12 +208,12 @@ public function getPartGeneratorTestCases()
];
$expectedSkip = $expected;
unset($expectedSkip[1], $expectedSkip[2], $expectedSkip[4]);
- $state = new UploadState([]);
+ $state = new DownloadState([]);
$state->setPartSize(2);
$stateSkip = clone $state;
- $stateSkip->markPartAsUploaded(1);
- $stateSkip->markPartAsUploaded(2);
- $stateSkip->markPartAsUploaded(4);
+ $stateSkip->markPartAsDownloaded(1);
+ $stateSkip->markPartAsDownloaded(2);
+ $stateSkip->markPartAsDownloaded(4);
return [
[true, $state, $expected],
[false, $state, $expected],
diff --git a/tests/Multipart/DownloadStateTest.php b/tests/Multipart/DownloadStateTest.php
index 8bf9ef2fd2..c368633a86 100644
--- a/tests/Multipart/DownloadStateTest.php
+++ b/tests/Multipart/DownloadStateTest.php
@@ -11,16 +11,11 @@ class DownloadStateTest extends TestCase
{
public function testCanManageStatusAndDownloadId()
{
- $state = new DownloadState(['a' => true]);
- $this->assertArrayHasKey('a', $state->getId());
+ $state = new DownloadState([]);
// Note: the state should not be initiated at first.
$this->assertFalse($state->isInitiated());
$this->assertFalse($state->isCompleted());
- $state->setUploadId('b', true);
- $this->assertArrayHasKey('b', $state->getId());
- $this->assertArrayHasKey('a', $state->getId());
-
$state->setStatus(DownloadState::INITIATED);
$this->assertFalse($state->isCompleted());
$this->assertTrue($state->isInitiated());
@@ -60,7 +55,6 @@ public function testSerializationWorks()
$state->setPartSize(5);
$state->markPartAsDownloaded(1);
$state->setStatus($state::INITIATED);
- $state->setUploadId('foo', 'bar');
$serializedState = serialize($state);
/** @var DownloadState $newState */
@@ -68,6 +62,5 @@ public function testSerializationWorks()
$this->assertSame(5, $newState->getPartSize());
$this->assertArrayHasKey(1, $state->getDownloadedParts());
$this->assertTrue($newState->isInitiated());
- $this->assertArrayHasKey('foo', $newState->getId());
}
-}
\ No newline at end of file
+}
diff --git a/tests/Multipart/TestDownloader.php b/tests/Multipart/TestDownloader.php
index 174e167ace..f0656f7569 100644
--- a/tests/Multipart/TestDownloader.php
+++ b/tests/Multipart/TestDownloader.php
@@ -12,26 +12,26 @@
*/
class TestDownloader extends AbstractDownloader
{
- public function __construct($client, $dest, array $config = [])
+ public function __construct($client, $source, array $config = [])
{
- $this->destStream = $this->createDestStream($dest);
- parent::__construct($client, $dest, $config + [
- 'bucket' => null,
- 'key' => null,
- 'exception_class' => S3MultipartDownloadException::class,
- ]);
+ $this->destStream = new Psr7\LazyOpenStream($source, 'w');
+ parent::__construct($client, $source, $config + [
+ 'bucket' => null,
+ 'key' => null,
+ 'exception_class' => S3MultipartDownloadException::class,
+ ]);
}
- protected function loadUploadWorkflowInfo()
+ protected function loadDownloadWorkflowInfo()
{
return [
'command' => [
'initiate' => 'GetObject',
- 'upload' => 'GetObject',
+ 'download' => 'GetObject'
],
'id' => [
'bucket' => 'Bucket',
'key' => 'Key',
- 'upload_id' => 'UploadId',
+ 'download_id' => 'DownloadId',
],
'part_num' => 'PartNumber',
];
@@ -42,47 +42,108 @@ protected function determinePartSize()
return $this->config['part_size'] ?: 2;
}
- protected function getInitiateParams($configType)
+ protected function getInitiateParams($type)
{
return [];
}
protected function createPart($seekable, $number)
{
-// if ($seekable) {
-// $body = Psr7\Utils::streamFor(fopen($this->source->getMetadata('uri'), 'r'));
-// $body = $this->limitPartStream($body);
-// } else {
-// $body = Psr7\Utils::streamFor($this->source->read($this->state->getPartSize()));
-// }
+ if ($seekable) {
+ $body = Psr7\Utils::streamFor(fopen($this->source->getMetadata('uri'), 'r'));
+ $body = $this->limitPartStream($body);
+ } else {
+ $body = Psr7\Utils::streamFor($this->source->read($this->state->getPartSize()));
+ }
// Do not create a part if the body size is zero.
-// if ($body->getSize() === 0) {
-// return false;
-// }
+ if ($body->getSize() === 0) {
+ return false;
+ }
return [
'PartNumber' => $number,
-// 'Body' => $body,
-// 'UploadId' => 'baz'
+ 'Body' => $body,
+ 'UploadId' => 'baz'
];
}
- protected function handleResult(CommandInterface $command, ResultInterface $result)
+ protected function handleResult($command, ResultInterface $result)
{
- $this->state->markPartAsUploaded($command['PartNumber'], [
- 'PartNumber' => $command['PartNumber'],
- 'ETag' => $result['ETag']
+ if (!($command instanceof CommandInterface)){
+ // single part downloads - part and range
+ $partNumber = 1;
+ $position = 0;
+ } elseif (!(isset($command['PartNumber']))) {
+ // multipart downloads - range
+ $seek = substr($command['Range'], strpos($command['Range'], "=") + 1);
+ $seek = (int)(strtok($seek, '-'));
+ $partNumber = $this->streamPositionArray[$seek];
+ $position = $seek;
+ } else {
+ // multipart downloads - part
+ $partNumber = $command['PartNumber'];
+ $position = $this->streamPositionArray[$command['PartNumber']];
+ }
+
+ $this->getState()->markPartAsDownloaded($partNumber, [
+ 'PartNumber' => $partNumber,
+ 'ETag' => $this->extractETag($result),
]);
+ $this->writeDestStream($position, $result['Body']);
}
- protected function getCompleteParams()
+ protected function extractETag(ResultInterface $result)
{
- return [
- 'MultipartUpload' => [
- 'Parts' => $this->state->getUploadedParts()
- ],
- 'UploadId' => 'baz'
- ];
+ return $result['ETag'];
}
+
+ protected function writeDestStream($position, $body)
+ {
+ $this->destStream->seek($position);
+ if ($body) {
+ $this->destStream->write($body->getContents());
+ }
+ }
+
+ protected function getDownloadType()
+ {
+ $config = $this->getConfig();
+ if (isset($config['partnumber'])) {
+ return ['config' => 'PartNumber',
+ 'configParam' => $config['partnumber']];
+ } elseif (isset($config['range'])) {
+ return ['config' => 'Range',
+ 'configParam' => $config['range']];
+ } elseif (isset($config['multipartdownloadtype']) && $config['multipartdownloadtype'] == 'Range') {
+ return ['config' => 'Range',
+ 'configParam' => 'bytes=0-'.MultipartDownloader::PART_MIN_SIZE,
+ 'multipart' => 'yes'
+ ];
+ } else {
+ return ['config' => 'PartNumber',
+ 'configParam' => 1,
+ 'multipart' => 'yes'];
+ }
+ }
+
+ public function setStreamPositionArray()
+ {
+ $parts = ceil($this->sourceSize/$this->state->getPartSize());
+ $position = 0;
+ if (isset($this->config['range']) or
+ (isset($this->config['multipartdownloadtype']) &&
+ $this->config['multipartdownloadtype'] == 'Range')) {
+ for ($i = 1; $i <= $parts; $i++) {
+ $this->streamPositionArray [$position] = $i;
+ $position += $this->state->getPartSize();
+ }
+ } else {
+ for ($i = 1; $i <= $parts; $i++) {
+ $this->streamPositionArray [$i] = $position;
+ $position += $this->state->getPartSize();
+ }
+ }
+ }
+
}
diff --git a/tests/S3/MultipartDownloaderTest.php b/tests/S3/MultipartDownloaderTest.php
index 3ea82ae9a7..e24433bbc3 100644
--- a/tests/S3/MultipartDownloaderTest.php
+++ b/tests/S3/MultipartDownloaderTest.php
@@ -6,6 +6,7 @@
use Aws\Result;
use Aws\S3\S3Client;
use Aws\Test\UsesServiceTrait;
+use Aws\S3\CalculatesChecksumTrait;
use GuzzleHttp\Promise;
use GuzzleHttp\Psr7;
use Psr\Http\Message\StreamInterface;
@@ -16,7 +17,14 @@
*/
class MultipartDownloaderTest extends TestCase
{
+// tests:
+// - workflow for each of the config options
+// - testing each method in multipart downloader
+// - createPart, setStreamPositionArray, createDestStream
+// -
+
use UsesServiceTrait;
+ use CalculatesChecksumTrait;
const MB = 1048576;
const FILENAME = '_aws-sdk-php-s3-mup-test-dots.txt';
@@ -30,19 +38,18 @@ public static function tear_down_after_class()
* @dataProvider getTestCases
*/
public function testS3MultipartDownloadWorkflow(
- array $clientOptions = [],
array $uploadOptions = [],
- $dest = null,
$error = false
) {
- $client = $this->getTestClient('s3', $clientOptions);
- $url = 'http://foo.s3.amazonaws.com/bar';
+ $client = $this->getTestClient('s3');
$this->addMockResults($client, [
- new Result(['UploadId' => 'baz']),
- new Result(['ETag' => 'A']),
- new Result(['ETag' => 'B']),
- new Result(['ETag' => 'C']),
- new Result(['Location' => $url])
+ new Result(['Body' => Psr7\Utils::streamFor(str_repeat('.', 10 * self::MB)),
+ 'ChecksumValidated' => 'CRC32',
+// 'ChecksumCRC32' =>
+// CalculatesChecksumTrait::getEncodedValue('crc32',
+// Psr7\Utils::streamFor(str_repeat('.', 10 * self::MB)))
+ 'ChecksumCRC32' => 'M6FqCg=='
+ ])
]);
if ($error) {
@@ -53,83 +60,84 @@ public function testS3MultipartDownloadWorkflow(
}
}
+ $filename = tmpfile();
+ $dest = stream_get_meta_data($filename)['uri'];
$downloader = new MultipartDownloader($client, $dest, $uploadOptions);
$result = $downloader->download();
+ $output = file_get_contents($dest);
+ $this->assertStringContainsString(str_repeat('.', 10 * self::MB), $output);
+ $this->assertTrue(filesize($dest) == 10*self::MB);
$this->assertTrue($downloader->getState()->isCompleted());
- $this->assertSame($url, $result['ObjectURL']);
}
public function getTestCases()
{
$defaults = [
'bucket' => 'foo',
- 'key' => 'bar',
+ 'key' => 'bar'
];
- $data = str_repeat('.', 12 * self::MB);
- $filename = sys_get_temp_dir() . '/' . self::FILENAME;
- file_put_contents($filename, $data);
-
return [
- [ // Seekable stream, regular config
- [],
- ['acl' => 'private'] + $defaults,
- Psr7\Utils::streamFor(fopen($filename, 'r'))
+ [
+ ['acl' => 'private'] + $defaults
],
- [ // Non-seekable stream
- [],
- $defaults,
- Psr7\Utils::streamFor($data)
+ [
+ ['MultipartDownloadType' => 'Range'] + $defaults
],
- [ // Error: bad part_size
- [],
- ['part_size' => 1] + $defaults,
- Psr7\FnStream::decorate(
- Psr7\Utils::streamFor($data), [
- 'getSize' => function () {return null;}
- ]
- ),
- 'InvalidArgumentException'
+ [
+ ['MultipartDownloadType' => 'Parts'] + $defaults
],
+ [
+ ['PartNumber' => '1'] + $defaults
+ ],
+ [
+ ['Range' => 'bytes=0-100'] + $defaults
+ ],
+ [
+ ['checksum_validation_enabled' => false] + $defaults
+ ],
+ [
+ ['checksum_validation_enabled' => true] + $defaults
+ ]
];
}
- public function testCanLoadStateFromService()
+ // continuing a prev download?
+ public function testCanLoadStateFromDownload()
{
$client = $this->getTestClient('s3');
- $url = 'http://foo.s3.amazonaws.com/bar';
$this->addMockResults($client, [
- new Result(['Parts' => [
- ['PartNumber' => 1, 'ETag' => 'A', 'Size' => 4 * self::MB],
- ]]),
- new Result(['ETag' => 'B']),
- new Result(['ETag' => 'C']),
- new Result(['Location' => $url])
+ new Result(['ETag' => 'A',
+ 'ChecksumValidated' => 'CRC32',
+ 'ContentLength' => 3 * self::MB])
]);
- $state = MultipartUploader::getStateFromService($client, 'foo', 'bar', 'baz');
- $source = Psr7\Utils::streamFor(str_repeat('.', 9 * self::MB));
- $uploader = new MultipartUploader($client, $source, ['state' => $state]);
- $result = $uploader->upload();
+ $size = 1 * self::MB;
+ $data = str_repeat('.', $size);
+ file_put_contents('php://memory', $data);
- $this->assertTrue($uploader->getState()->isCompleted());
- $this->assertSame(4 * self::MB, $uploader->getState()->getPartSize());
- $this->assertSame($url, $result['ObjectURL']);
+ $state = MultipartDownloader::getStateFromService($client, 'foo', 'bar', 'php://memory');
+ $downloader = new MultipartDownloader($client, $dest, ['state' => $state]);
+ $downloader->download();
+
+ $this->assertTrue($downloader->getState()->isCompleted());
+// $this->assertSame(4 * self::MB, $downloader->getState()->getPartSize());
+// $this->assertSame($url, $result['ObjectURL']);
}
public function testCanUseCaseInsensitiveConfigKeys()
{
$client = $this->getTestClient('s3');
- $putObjectMup = new MultipartUploader($client, Psr7\Utils::streamFor('x'), [
+ $putObjectMup = new MultipartDownloader($client, 'php://temp', [
'Bucket' => 'bucket',
'Key' => 'key',
]);
- $classicMup = new MultipartUploader($client, Psr7\Utils::streamFor('x'), [
+ $classicMup = new MultipartDownloader($client, 'php://temp', [
'bucket' => 'bucket',
'key' => 'key',
]);
- $configProp = (new \ReflectionClass(MultipartUploader::class))
+ $configProp = (new \ReflectionClass(MultipartDownloader::class))
->getProperty('config');
$configProp->setAccessible(true);
@@ -146,11 +154,11 @@ public function testMultipartSuccessStreams()
return [
[ // Seekable stream, regular config
- Psr7\Utils::streamFor(fopen($filename, 'r')),
+ 'php://temp',
$size,
],
[ // Non-seekable stream
- Psr7\Utils::streamFor($data),
+ 'php://temp',
$size,
]
];
@@ -159,14 +167,14 @@ public function testMultipartSuccessStreams()
/**
* @dataProvider testMultipartSuccessStreams
*/
- public function testS3MultipartUploadParams($stream, $size)
+ public function testS3MultipartDownloadParams($dest, $size)
{
/** @var \Aws\S3\S3Client $client */
$client = $this->getTestClient('s3');
$client->getHandlerList()->appendSign(
Middleware::tap(function ($cmd, $req) {
$name = $cmd->getName();
- if ($name === 'UploadPart') {
+ if ($name === 'GetObject') {
$this->assertTrue(
$req->hasHeader('Content-MD5')
);
@@ -177,36 +185,40 @@ public function testS3MultipartUploadParams($stream, $size)
'bucket' => 'foo',
'key' => 'bar',
'add_content_md5' => true,
- 'params' => [
- 'RequestPayer' => 'test',
- 'ContentLength' => $size
- ],
- 'before_initiate' => function($command) {
- $this->assertSame('test', $command['RequestPayer']);
- },
- 'before_upload' => function($command) use ($size) {
- $this->assertLessThan($size, $command['ContentLength']);
- $this->assertSame('test', $command['RequestPayer']);
- },
- 'before_complete' => function($command) {
- $this->assertSame('test', $command['RequestPayer']);
- }
+// 'params' => [
+// 'RequestPayer' => 'test',
+// 'ContentLength' => $size
+// ],
+// 'before_initiate' => function($command) {
+// $this->assertSame('test', $command['RequestPayer']);
+// },
+// 'before_download' => function($command) use ($size) {
+// $this->assertLessThan($size, $command['ContentLength']);
+// $this->assertSame('test', $command['RequestPayer']);
+// },
+// 'before_complete' => function($command) {
+// $this->assertSame('test', $command['RequestPayer']);
+// },
+ 'checksum_validation_enabled' => false
];
$url = 'http://foo.s3.amazonaws.com/bar';
$this->addMockResults($client, [
- new Result(['UploadId' => 'baz']),
- new Result(['ETag' => 'A']),
- new Result(['ETag' => 'B']),
- new Result(['ETag' => 'C']),
- new Result(['Location' => $url])
+ new Result(['PartNumber' => 1, 'ETag' => 'A', 'Body' => 'foobar',
+ 'ChecksumValidated' => 'CRC32',
+// 'ChecksumCRC32' => CalculatesChecksumTrait::getEncodedValue('crc32', 'foobar')
+]),
+ new Result(['PartNumber' => 2, 'ETag' => 'B', 'Body' => 'foobar2',
+ 'ChecksumValidated' => 'CRC32',
+// 'ChecksumCRC32' => CalculatesChecksumTrait::getEncodedValue('crc32', 'foobar2')
+ ])
]);
-
- $uploader = new MultipartUploader($client, $stream, $uploadOptions);
- $result = $uploader->upload();
-
+ $filename = tmpfile();
+ $dest = stream_get_meta_data($filename)['uri'];
+ $uploader = new MultipartDownloader($client, $dest, $uploadOptions);
+ $result = $uploader->download();
+ print_r($result);
$this->assertTrue($uploader->getState()->isCompleted());
- $this->assertSame($url, $result['ObjectURL']);
}
public function getContentTypeSettingTests()
@@ -271,8 +283,8 @@ public function testS3MultipartContentTypeSetting(
new Result(['Location' => $url])
]);
- $uploader = new MultipartUploader($client, $stream, $uploadOptions);
- $result = $uploader->upload();
+ $uploader = new MultipartDownloader($client, $stream, $uploadOptions);
+ $result = $uploader->download();
$this->assertTrue($uploader->getState()->isCompleted());
$this->assertSame($url, $result['ObjectURL']);
@@ -280,8 +292,8 @@ public function testS3MultipartContentTypeSetting(
public function testAppliesAmbiguousSuccessParsing()
{
- $this->expectExceptionMessage("An exception occurred while uploading parts to a multipart upload");
- $this->expectException(\Aws\S3\Exception\S3MultipartUploadException::class);
+ $this->expectExceptionMessage("An exception occurred while downloading parts to a multipart download");
+ $this->expectException(\Aws\S3\Exception\S3MultipartDownloadException::class);
$counter = 0;
$httpHandler = function ($request, array $options) use (&$counter) {
@@ -303,63 +315,22 @@ public function testAppliesAmbiguousSuccessParsing()
'http_handler' => $httpHandler
]);
- $data = str_repeat('.', 12 * 1048576);
- $source = Psr7\Utils::streamFor($data);
-
- $uploader = new MultipartUploader(
- $s3,
- $source,
- [
- 'bucket' => 'test-bucket',
- 'key' => 'test-key'
- ]
- );
- $uploader->upload();
- }
-
- public function testFailedUploadPrintsPartialProgressBar()
- {
- $partialBar = [ "Transfer initiated...\n| | 0.0%\n",
- "|== | 12.5%\n",
- "|===== | 25.0%\n"];
- $this->expectOutputString("{$partialBar[0]}{$partialBar[1]}{$partialBar[2]}");
-
- $this->expectExceptionMessage("An exception occurred while uploading parts to a multipart upload");
- $this->expectException(\Aws\S3\Exception\S3MultipartUploadException::class);
- $counter = 0;
-
- $httpHandler = function ($request, array $options) use (&$counter) {
- if ($counter < 4) {
- $body = "baz";
- } else {
- $body = "\n\n\n";
- }
- $counter++;
-
- return Promise\Create::promiseFor(
- new Psr7\Response(200, [], $body)
- );
- };
-
- $s3 = new S3Client([
- 'version' => 'latest',
- 'region' => 'us-east-1',
- 'http_handler' => $httpHandler
- ]);
+// $data = str_repeat('.', 12 * 1048576);
+// $source = Psr7\Utils::streamFor($data);
- $data = str_repeat('.', 50 * self::MB);
- $source = Psr7\Utils::streamFor($data);
+ $filename = tmpfile();
+ $dest = stream_get_meta_data($filename)['uri'];
- $uploader = new MultipartUploader(
+ $downloader = new MultipartDownloader(
$s3,
- $source,
+ $dest,
[
'bucket' => 'test-bucket',
'key' => 'test-key',
- 'track_upload' => 'true'
+ 'checksum_validation_enabled' => false
]
);
- $uploader->upload();
+ $downloader->download();
}
}