From 4b07044a96999bc6d8009e0b5f626fb0d2a56aa4 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Tue, 21 Aug 2018 15:26:42 +0800 Subject: [PATCH 01/12] add decrypt get range data feature in CryptoBucket --- oss2/api.py | 10 ++++++++-- oss2/crypto.py | 10 +++++----- oss2/models.py | 29 +++++++++++++++++++++++------ oss2/utils.py | 22 ++++++++++++++++------ 4 files changed, 52 insertions(+), 19 deletions(-) diff --git a/oss2/api.py b/oss2/api.py index 8daf5677..7eafdfab 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -1703,6 +1703,7 @@ def put_object_from_file(self, key, filename, return self.put_object(key, f, headers=headers, progress_callback=progress_callback) def get_object(self, key, + byte_range=None, headers=None, progress_callback=None, params=None): @@ -1715,6 +1716,7 @@ def get_object(self, key, 'hello world' :param key: 文件名 + :param byte_range: 指定下载范围。参见 :ref:`byte_range` :param headers: HTTP头部 :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict @@ -1730,8 +1732,12 @@ def get_object(self, key, """ headers = http.CaseInsensitiveDict(headers) - if 'range' in headers: - raise ClientError('Crypto bucket do not support range get') + if byte_range and (not utils.is_multiple_sizeof_encrypt_block(byte_range[0])): + raise ClientError('Crypto bucket get range start must align to encrypt block') + + range_string = _make_range_string(byte_range) + if range_string: + headers['range'] = range_string encrypted_result = self.bucket.get_object(key, headers=headers, params=params, progress_callback=None) diff --git a/oss2/crypto.py b/oss2/crypto.py index 66faa1c3..c2691083 100644 --- a/oss2/crypto.py +++ b/oss2/crypto.py @@ -35,11 +35,11 @@ def __init__(self, cipher): self.plain_start = None self.cipher = cipher - def make_encrypt_adapter(self, stream, key, start): - return utils.make_cipher_adapter(stream, partial(self.cipher.encrypt, self.cipher(key, start))) + def make_encrypt_adapter(self, stream, key, count_start, count_offset=0): + return utils.make_cipher_adapter(stream, partial(self.cipher.encrypt, self.cipher(key, count_start, count_offset))) - def make_decrypt_adapter(self, stream, key, start): - return utils.make_cipher_adapter(stream, partial(self.cipher.decrypt, self.cipher(key, start))) + def make_decrypt_adapter(self, stream, key, count_start, count_offset=0): + return utils.make_cipher_adapter(stream, partial(self.cipher.decrypt, self.cipher(key, count_start, count_offset))) _LOCAL_RSA_TMP_DIR = '.oss-local-rsa' @@ -248,4 +248,4 @@ def decrypt_oss_meta_data(self, headers, key, conv=lambda x: x): except OssError as e: raise e except: - return None \ No newline at end of file + return None diff --git a/oss2/models.py b/oss2/models.py index 2f61dde2..e5db9fa3 100644 --- a/oss2/models.py +++ b/oss2/models.py @@ -7,7 +7,8 @@ 该模块包含Python SDK API接口所需要的输入参数以及返回值类型。 """ -from .utils import http_to_unixtime, make_progress_adapter, make_crc_adapter +from .utils import http_to_unixtime, make_progress_adapter, make_crc_adapter, \ + calc_aes_ctr_offset_by_range, is_multiple_sizeof_encrypt_block from .exceptions import ClientError, InconsistentError from .compat import urlunquote, to_string from .select_response import SelectResponseAdapter @@ -127,8 +128,11 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi self.__crc_enabled = crc_enabled self.__crypto_provider = crypto_provider - if _hget(resp.headers, 'x-oss-meta-oss-crypto-key') and _hget(resp.headers, 'Content-Range'): - raise ClientError('Could not get an encrypted object using byte-range parameter') + content_range = _hget(resp.headers, 'Content-Range') + if _hget(resp.headers, 'x-oss-meta-oss-crypto-key') and content_range: + byte_range = self._parse_range_str(content_range) + if not is_multiple_sizeof_encrypt_block(byte_range[0]): + raise ClientError('Could not get an encrypted object using byte-range parameter') if progress_callback: self.stream = make_progress_adapter(self.resp, progress_callback, self.content_length) @@ -140,14 +144,27 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi if self.__crypto_provider: key = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, 'x-oss-meta-oss-crypto-key') - start = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, 'x-oss-meta-oss-crypto-start') + count_start = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, 'x-oss-meta-oss-crypto-start') + + # if content range , adjust the decrypt adapter + count_offset = 0; + if content_range: + byte_range = self._parse_range_str(content_range) + count_offset = calc_aes_ctr_offset_by_range(byte_range[0]) + cek_alg = _hget(resp.headers, 'x-oss-meta-oss-cek-alg') - if key and start and cek_alg: - self.stream = self.__crypto_provider.make_decrypt_adapter(self.stream, key, start) + if key and count_start and cek_alg: + self.stream = self.__crypto_provider.make_decrypt_adapter(self.stream, key, count_start, count_offset) else: raise InconsistentError('all metadata keys are required for decryption (x-oss-meta-oss-crypto-key, \ x-oss-meta-oss-crypto-start, x-oss-meta-oss-cek-alg)', self.request_id) + def _parse_range_str(self, content_range): + # :param str content_range: sample 'bytes 0-128/1024' + range_data = (content_range.split(' ',2)[1]).split('/',2)[0] + range_start, range_end = range_data.split('-',2) + return (int(range_start), int(range_end)) + def read(self, amt=None): return self.stream.read(amt) diff --git a/oss2/utils.py b/oss2/utils.py index 8c80d702..8d5e44ee 100644 --- a/oss2/utils.py +++ b/oss2/utils.py @@ -588,11 +588,20 @@ def random_counter(begin=1, end=10): # aes 256, key always is 32 bytes _AES_256_KEY_SIZE = 32 -_AES_CTR_COUNTER_BITS_LEN = 8 * 16 +_AES_CTR_COUNTER_LEN = 16 +_AES_CTR_COUNTER_BITS_LEN = 8 * _AES_CTR_COUNTER_LEN _AES_GCM = 'AES/GCM/NoPadding' +def is_multiple_sizeof_encrypt_block(byte_range_start): + return (byte_range_start % _AES_CTR_COUNTER_LEN == 0) + +def calc_aes_ctr_offset_by_range(byte_range_start): + if not is_multiple_sizeof_encrypt_block(byte_range_start): + raise ClientError('Invalid get range value for client encryption') + return byte_range_start / _AES_CTR_COUNTER_LEN + class AESCipher: """AES256 加密实现。 :param str key: 对称加密数据密钥 @@ -613,15 +622,16 @@ def get_key(): def get_start(): return random_counter() - def __init__(self, key=None, start=None): + def __init__(self, key=None, count_start=None, count_offset=0): self.key = key + self.count_offset = count_offset if not self.key: self.key = random_aes256_key() - if not start: - self.start = random_counter() + if not count_start: + self.count_start = random_counter() else: - self.start = int(start) - ctr = Counter.new(_AES_CTR_COUNTER_BITS_LEN, initial_value=self.start) + self.count_start = int(count_start) + ctr = Counter.new(_AES_CTR_COUNTER_BITS_LEN, initial_value=(self.count_start + self.count_offset)) self.__cipher = AES.new(self.key, AES.MODE_CTR, counter=ctr) def encrypt(self, raw): From 7c908527d0445b54bbab9fa848404e47065d9d61 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Tue, 21 Aug 2018 16:21:49 +0800 Subject: [PATCH 02/12] add unittest for get range --- unittests/test_models.py | 16 ++++++++++++++++ unittests/test_utils.py | 20 ++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 unittests/test_models.py create mode 100644 unittests/test_utils.py diff --git a/unittests/test_models.py b/unittests/test_models.py new file mode 100644 index 00000000..25cea838 --- /dev/null +++ b/unittests/test_models.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import unittest +from oss2.models import * +from unittests.common import * + + +class TestModels(unittest.TestCase): + def test_parse_range_str(self): + resp = do4body('', 0, body='') + get_obj_result = GetObjectResult(resp) + + content_range = 'bytes 0-128/1024' + range_data = get_obj_result._parse_range_str(content_range) + self.assertEqual(range_data[0], 0) + self.assertEqual(range_data[1], 128) diff --git a/unittests/test_utils.py b/unittests/test_utils.py new file mode 100644 index 00000000..ff0a5bac --- /dev/null +++ b/unittests/test_utils.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +import unittest +from oss2.utils import * + + +class TestUtils(unittest.TestCase): + def test_is_multiple_sizeof_encrypt_block(self): + byte_range_start = 1024 + is_multiple = is_multiple_sizeof_encrypt_block(byte_range_start) + self.assertTrue(is_multiple) + + byte_range_start = 1025 + is_multiple = is_multiple_sizeof_encrypt_block(byte_range_start) + self.assertFalse(is_multiple) + + def test_calc_aes_ctr_offset_by_range(self): + byte_range_start = 1024 + cout_offset = calc_aes_ctr_offset_by_range(byte_range_start) + self.assertEqual(cout_offset, 1024 / 16) From 2ac86a63c6caf666d15bd7b1207497af5845cb89 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Tue, 21 Aug 2018 18:24:42 +0800 Subject: [PATCH 03/12] add tests of rsa and kms crypto range get --- oss2/utils.py | 2 ++ tests/test_object.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/oss2/utils.py b/oss2/utils.py index 8d5e44ee..f8aec5ba 100644 --- a/oss2/utils.py +++ b/oss2/utils.py @@ -595,6 +595,8 @@ def random_counter(begin=1, end=10): def is_multiple_sizeof_encrypt_block(byte_range_start): + if byte_range_start is None: + byte_range_start = 0 return (byte_range_start % _AES_CTR_COUNTER_LEN == 0) def calc_aes_ctr_offset_by_range(byte_range_start): diff --git a/tests/test_object.py b/tests/test_object.py index 31d22417..08dd214a 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -84,6 +84,27 @@ def assert_result(result): self.assertTrue(get_result.server_crc is not None) self.assertTrue(get_result.client_crc == get_result.server_crc) + def test_rsa_crypto_range_get(self): + key = self.random_key() + content = random_bytes(1024) + + self.rsa_crypto_bucket.put_object(key, content) + + get_result = self.rsa_crypto_bucket.get_object(key, byte_range=(None, None)) + self.assertEqual(get_result.read(), content[:]) + + get_result = self.rsa_crypto_bucket.get_object(key, byte_range=(32, None)) + self.assertEqual(get_result.read(), content[32:]) + + get_result = self.rsa_crypto_bucket.get_object(key, byte_range=(None, 32)) + self.assertEqual(get_result.read(), content[-32:]) + + get_result = self.rsa_crypto_bucket.get_object(key, byte_range=(32, 103)) + self.assertEqual(get_result.read(), content[32:103+1]) + + self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(31, None)) + self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(None, 31)) + def test_kms_crypto_object(self): if is_py33: return @@ -116,6 +137,31 @@ def assert_result(result): self.assertTrue(get_result.server_crc is not None) self.assertTrue(get_result.client_crc == get_result.server_crc) + def test_kms_crypto_range_get(self): + if is_py33: + return + + key = self.random_key() + content = random_bytes(1024) + + self.kms_crypto_bucket.put_object(key, content, headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))}) + + get_result = self.kms_crypto_bucket.get_object(key, byte_range=(None, None)) + self.assertEqual(get_result.read(), content[:]) + + get_result = self.kms_crypto_bucket.get_object(key, byte_range=(32, None)) + self.assertEqual(get_result.read(), content[32:]) + + get_result = self.kms_crypto_bucket.get_object(key, byte_range=(None, 32)) + self.assertEqual(get_result.read(), content[-32:]) + + get_result = self.kms_crypto_bucket.get_object(key, byte_range=(32, 103)) + self.assertEqual(get_result.read(), content[32:103+1]) + + self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(31, None)) + self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(None, 31)) + def test_restore_object(self): auth = oss2.Auth(OSS_ID, OSS_SECRET) bucket = oss2.Bucket(auth, OSS_ENDPOINT, random_string(63).lower()) From ce4199c9f304e63fd8ffe91a83bf8748cbacf848 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Tue, 21 Aug 2018 18:37:03 +0800 Subject: [PATCH 04/12] add CryptoBucket get range example --- examples/object_crypto.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/examples/object_crypto.py b/examples/object_crypto.py index 5576abd7..936d1864 100644 --- a/examples/object_crypto.py +++ b/examples/object_crypto.py @@ -62,6 +62,15 @@ os.remove(filename) +# 下载部分文件 +result = bucket.get_object(key, byte_range=(32,1024)) + +#验证一下 +content_got = b'' +for chunk in result: + content_got +=chunk +assert content_got == content[32:1025] + # 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS)，此模式下只提供对象整体上传下载操作 bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, @@ -93,4 +102,13 @@ with open(filename, 'rb') as fileobj: assert fileobj.read() == content -os.remove(filename) \ No newline at end of file +os.remove(filename) + +# 下载部分文件 +result = bucket.get_object(key, byte_range=(32,1024)) + +#验证一下 +content_got = b'' +for chunk in result: + content_got +=chunk +assert content_got == content[32:1025] From c7a05513a72f26a7d608d736fa680057c23cc288 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Mon, 8 Oct 2018 17:42:00 +0800 Subject: [PATCH 05/12] add crypto Bucket multipart upload feature --- oss2/api.py | 162 ++++++++++++++++++++++++++++++++++++++++ oss2/crypto.py | 53 +++++++++---- oss2/headers.py | 17 +++++ oss2/models.py | 36 +++++++-- oss2/resumable.py | 5 +- oss2/utils.py | 43 ++++++++--- unittests/test_utils.py | 4 +- 7 files changed, 282 insertions(+), 38 deletions(-) diff --git a/oss2/api.py b/oss2/api.py index 7eafdfab..fe1b269b 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -177,6 +177,8 @@ def progress_callback(bytes_consumed, total_bytes): from .crypto import BaseCryptoProvider from .headers import * +from .utils import calc_aes_ctr_offset_by_data_offset, is_valid_crypto_part_size, determine_crypto_part_size + import time import shutil import base64 @@ -1646,6 +1648,7 @@ def __init__(self, auth, endpoint, bucket_name, crypto_provider, self.enable_crc = enable_crc self.bucket = Bucket(auth, endpoint, bucket_name, is_cname, session, connect_timeout, app_name, enable_crc=False) + self.multipart_upload_contexts = {} def put_object(self, key, data, headers=None, @@ -1774,6 +1777,165 @@ def get_object_to_file(self, key, filename, return result + def init_multipart_upload_securely(self, key, data_size, part_size = None, headers=None): + """客户端加密初始化分片上传。 + + 返回值中的 `upload_id` 以及Bucket名和Object名三元组唯一对应了此次分片上传事件。 + 返回值中的 `part_size` 限制了后续分片上传中除最后一个分片之外其他分片大小必须一致 + 返回值中的 `part_number` 限制了后续分片上传分片总数目，未完全上传不允许complete操作 + + :param str key: 待上传的文件名 + :param int data_size : 待上传文件总大小 + :param int part_size : 后续分片上传时除最后一个分片之外的其他分片大小 + + :param headers: HTTP头部 + :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict + + :return: :class:`InitMultipartUploadResult ` + """ + if part_size is not None: + res = is_valid_crypto_part_size(part_size, data_size) + if not res: + raise ClientError("Crypto bucket get an invalid part_size") + else: + part_size = determine_crypto_part_size(data_size) + + logger.info("Start to init multipart upload securely, data_size: {0}, part_size: {1}".format(data_size, part_size)) + + crypto_key = self.crypto_provider.get_key() + crypto_start = self.crypto_provider.get_start() + + part_number = (data_size - 1) / part_size + 1 + context = CryptoMultipartContext(crypto_key, crypto_start, part_size, part_number, data_size) + + headers = self.crypto_provider.build_header(headers, context) + + resp = self.bucket.init_multipart_upload(key, headers) + resp.part_size = context.part_size + resp.part_number = context.part_number + + context.upload_id = resp.upload_id + self.multipart_upload_contexts[resp.upload_id] = context + + logger.info("Init multipart upload securely done, upload_id = {0} put into local contexts".format(context.upload_id)) + + return resp + + def upload_part_securely(self, key, upload_id, part_number, data, progress_callback=None, headers=None): + """客户端加密上传一个分片。 + + :param str key: 待上传文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 + :param str upload_id: 分片上传ID + :param int part_number: 分片号，最小值是1. + :param data: 待上传数据。 + :param progress_callback: 用户指定进度回调函数。可以用来实现进度条等功能。参考 :ref:`progress_callback` 。 + + :param headers: 用户指定的HTTP头部。可以指定Content-MD5头部等 + :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict + + :return: :class:`PutObjectResult ` + """ + logger.info("Start upload part securely, upload_id = {0}, part_number = {1}".format(upload_id, part_number)) + try: + context = self.multipart_upload_contexts[upload_id] + except: + raise ClientError("Crypto bucket can't find the upload_id in local contexts") + + if len(data) != context.part_size and part_number != context.part_number: + raise ClientError("Please upload part with correct size unless the last part") + + crypto_key = context.crypto_key + start = context.crypto_start + offset = context.part_size * (part_number - 1) + count_offset = utils.calc_aes_ctr_offset_by_data_offset(offset) + + data = self.crypto_provider.make_encrypt_adapter(data, crypto_key, start, count_offset=count_offset) + if self.enable_crc: + data = utils.make_crc_adapter(data) + + resp = self.bucket.upload_part(key, upload_id, part_number, data, progress_callback, headers) + + context.uploaded_parts.add(part_number) + logger.info("Upload part securely done, the part {0} already put into local contexts".format(part_number)) + + return resp + + + def complete_multipart_upload_securely(self, key, upload_id, parts, headers=None): + """客户端加密完成分片上传，创建文件。 + 当所有分片均已上传成功，才可以调用此函数 + + :param str key: 待上传的文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 + :param str upload_id: 分片上传ID + + :param parts: PartInfo列表。PartInfo中的part_number和etag是必填项。其中的etag可以从 :func:`upload_part` 的返回值中得到。 + :type parts: list of `PartInfo ` + + :param headers: HTTP头部 + :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict + + :return: :class:`PutObjectResult ` + """ + logger.info("Start complete multipart upload securely, upload_id = {0}".format(upload_id)) + try: + context = self.multipart_upload_contexts[upload_id] + except: + raise ClientError("Crypto bucket can't find the upload_id in local contexts") + + if len(context.uploaded_parts) != context.part_number: + raise ClientError("Incomplete parts uploaded in local contexts") + + res = self.bucket.complete_multipart_upload(key, upload_id, parts, headers) + del self.multipart_upload_contexts[upload_id] + + logger.info("Complete multipart upload securely done, upload_id = {0} remove from local contexts".format(upload_id)) + + return res + + def abort_multipart_upload_securely(self, key, upload_id): + """取消分片上传。 + + :param str key: 待上传的文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 + :param str upload_id: 分片上传ID + + :return: :class:`RequestResult ` + """ + logger.info("Start abort multipart upload securely, upload_id = {0}".format(upload_id)) + try: + context = self.multipart_upload_contexts[upload_id] + except: + raise ClientError("Crypto bucket can't find the upload_id in local contexts") + + res = self.bucket.abort_multipart_upload_securely(key, upload_id) + del self.multipart_upload_contexts[upload_id] + + logger.info("Abort multipart upload securely done, upload_id = {0} remove from local contexts".format(upload_id)) + + return res + + def list_parts_securely(self, key, upload_id, + marker='', max_parts=1000): + """列举已经上传的分片。支持分页。 + + :param str key: 文件名 + :param str upload_id: 分片上传ID + :param str marker: 分页符 + :param int max_parts: 一次最多罗列多少分片 + + :return: :class:`ListPartsResult ` + """ + logger.info("Start list parts securely, upload_id = {0}".format(upload_id)) + try: + context = self.multipart_upload_contexts[upload_id] + except: + raise ClientError("Crypto bucket can't find the upload_id in local contexts") + + res = self.bucket.list_parts(key, upload_id, marker = marker, max_parts = max_parts) + logger.info("List parts securely done, upload_id = {0}".format(upload_id)) + return res + + def determine_valid_part_size(data_size): + return determine_crypto_part_size(data_size) def _normalize_endpoint(endpoint): if not endpoint.startswith('http://') and not endpoint.startswith('https://'): diff --git a/oss2/crypto.py b/oss2/crypto.py index c2691083..d4397024 100644 --- a/oss2/crypto.py +++ b/oss2/crypto.py @@ -13,6 +13,7 @@ from . import utils from .compat import to_string, to_bytes, to_unicode from .exceptions import OssError, ClientError, OpenApiFormatError, OpenApiServerError +from .headers import * from Crypto.Cipher import PKCS1_OAEP from Crypto.PublicKey import RSA @@ -41,6 +42,8 @@ def make_encrypt_adapter(self, stream, key, count_start, count_offset=0): def make_decrypt_adapter(self, stream, key, count_start, count_offset=0): return utils.make_cipher_adapter(stream, partial(self.cipher.decrypt, self.cipher(key, count_start, count_offset))) + def check_plain_key_valid(self, plain_key, plain_key_hmac): + pass _LOCAL_RSA_TMP_DIR = '.oss-local-rsa' @@ -88,22 +91,27 @@ def __init__(self, dir=None, key='', passphrase=None, cipher=utils.AESCipher): except (ValueError, TypeError, IndexError) as e: raise ClientError(str(e)) - def build_header(self, headers=None): + def build_header(self, headers=None, multipart_context=None): if not isinstance(headers, CaseInsensitiveDict): headers = CaseInsensitiveDict(headers) if 'content-md5' in headers: - headers['x-oss-meta-unencrypted-content-md5'] = headers['content-md5'] + headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] del headers['content-md5'] if 'content-length' in headers: - headers['x-oss-meta-unencrypted-content-length'] = headers['content-length'] + headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] del headers['content-length'] - headers['x-oss-meta-oss-crypto-key'] = b64encode_as_string(self.__encrypt_obj.encrypt(self.plain_key)) - headers['x-oss-meta-oss-crypto-start'] = b64encode_as_string(self.__encrypt_obj.encrypt(to_bytes(str(self.plain_start)))) - headers['x-oss-meta-oss-cek-alg'] = self.cipher.ALGORITHM - headers['x-oss-meta-oss-wrap-alg'] = 'rsa' + headers[OSS_CLIENT_SIDE_CRYPTO_KEY] = b64encode_as_string(self.__encrypt_obj.encrypt(self.plain_key)) + headers[OSS_CLIENT_SIDE_CRYPTO_START] = b64encode_as_string(self.__encrypt_obj.encrypt(to_bytes(str(self.plain_start)))) + headers[OSS_CLIENT_SIDE_CRYPTO_CEK_ALG] = self.cipher.ALGORITHM + headers[OSS_CLIENT_SIDE_CRYPTO_WRAP_ALG] = 'rsa' + + headers[OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC] = b64encode_as_string(str(hash(self.plain_key))) + # multipart file build header + if multipart_context: + headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH] = str(multipart_context.data_size) self.plain_key = None self.plain_start = None @@ -120,10 +128,16 @@ def get_start(self): def decrypt_oss_meta_data(self, headers, key, conv=lambda x:x): try: - return conv(self.__decrypt_obj.decrypt(utils.b64decode_from_string(headers[key]))) + if key == OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC: + return conv(utils.b64decode_from_string(headers[key])) + else: + return conv(self.__decrypt_obj.decrypt(utils.b64decode_from_string(headers[key]))) except: return None + def check_plain_key_valid(self, plain_key, plain_key_hmac): + if str(hash(plain_key)) != plain_key_hmac: + raise ClientError("The decrypted key is inconsistent, make sure use right RSA key pair") class AliKMSProvider(BaseCryptoProvider): """使用aliyun kms服务加密数据密钥。kms的详细说明参见 @@ -152,21 +166,26 @@ def __init__(self, access_key_id, access_key_secret, region, cmkey, sts_token = self.encrypted_key = None - def build_header(self, headers=None): + def build_header(self, headers=None, multipart_context=None): if not isinstance(headers, CaseInsensitiveDict): headers = CaseInsensitiveDict(headers) + if 'content-md5' in headers: - headers['x-oss-meta-unencrypted-content-md5'] = headers['content-md5'] + headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] del headers['content-md5'] if 'content-length' in headers: - headers['x-oss-meta-unencrypted-content-length'] = headers['content-length'] + headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] del headers['content-length'] - headers['x-oss-meta-oss-crypto-key'] = self.encrypted_key - headers['x-oss-meta-oss-crypto-start'] = self.__encrypt_data(to_bytes(str(self.plain_start))) - headers['x-oss-meta-oss-cek-alg'] = self.cipher.ALGORITHM - headers['x-oss-meta-oss-wrap-alg'] = 'kms' + headers[OSS_CLIENT_SIDE_CRYPTO_KEY] = self.encrypted_key + headers[OSS_CLIENT_SIDE_CRYPTO_START] = self.__encrypt_data(to_bytes(str(self.plain_start))) + headers[OSS_CLIENT_SIDE_CRYPTO_CEK_ALG] = self.cipher.ALGORITHM + headers[OSS_CLIENT_SIDE_CRYPTO_WRAP_ALG] = 'kms' + + # multipart file build header + if multipart_context: + headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH] = str(multipart_context.data_size) self.encrypted_key = None self.plain_start = None @@ -249,3 +268,7 @@ def decrypt_oss_meta_data(self, headers, key, conv=lambda x: x): raise e except: return None + + def check_plain_key_valid(self, plain_key, plain_key_hmac): + if plain_key_hmac: + raise ClientError('AliKMSProvider not support check_plain_key_valid') diff --git a/oss2/headers.py b/oss2/headers.py index 5ddf76ca..9508f740 100644 --- a/oss2/headers.py +++ b/oss2/headers.py @@ -30,6 +30,23 @@ OSS_SERVER_SIDE_ENCRYPTION = "x-oss-server-side-encryption" OSS_SERVER_SIDE_ENCRYPTION_KEY_ID = "x-oss-server-side-encryption-key-id" +OSS_CLIENT_SIDE_CRYPTO_KEY = "x-oss-meta-oss-crypto-key" +OSS_CLIENT_SIDE_CRYPTO_START = "x-oss-meta-oss-crypto-start" +OSS_CLIENT_SIDE_CRYPTO_CEK_ALG = "x-oss-meta-oss-cek-alg" +OSS_CLIENT_SIDE_CRYPTO_WRAP_ALG = "x-oss-meta-oss-wrap-alg" +OSS_CLIENT_SIDE_CRYTPO_MATDESC = "x-oss-meta-oss-crypto-matdesc" +OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH = "x-oss-meta-oss-crypto-unencrypted-content-length" +OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_MD5 = "x-oss-meta-oss-crypto-unencrypted-content-md5" +OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC = "x-oss-meta-oss-crypto-key-hmac" + +#OSS_CLIENT_SIDE_CRYPTO_KEY = "x-oss-client-side-crypto-key" +#OSS_CLIENT_SIDE_CRYPTO_START = "x-oss-client-side-crypto-start" +#OSS_CLIENT_SIDE_CRYPTO_CEK_ALG = "x-oss-client-side-crypto-cek-alg" +#OSS_CLIENT_SIDE_CRYPTO_WRAP_ALG = "x-oss-client-side-crypto-wrap-alg" +#OSS_CLIENT_SIDE_CRYTPO_MATDESC = "x-oss-client-side-crypto-matdesc" +#OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH = "x-oss-client-side-crypto-unencrypted-content-length" +#OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_MD5 = "x-oss-client-side-crypto-unencrypted-content-md5" +#OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC = "x-oss-client-side-crypto-key-hmac" class RequestHeader(dict): def __init__(self, *arg, **kw): diff --git a/oss2/models.py b/oss2/models.py index e5db9fa3..fd42d441 100644 --- a/oss2/models.py +++ b/oss2/models.py @@ -8,7 +8,7 @@ """ from .utils import http_to_unixtime, make_progress_adapter, make_crc_adapter, \ - calc_aes_ctr_offset_by_range, is_multiple_sizeof_encrypt_block + calc_aes_ctr_offset_by_data_offset, is_multiple_sizeof_encrypt_block from .exceptions import ClientError, InconsistentError from .compat import urlunquote, to_string from .select_response import SelectResponseAdapter @@ -34,6 +34,17 @@ def __init__(self, part_number, etag, size=None, last_modified=None, part_crc=No self.last_modified = last_modified self.part_crc = part_crc +class CryptoMultipartContext(object): + """表示客户端加密文件通过Multipart接口上传的meta信息 + """ + def __init__(self, crypto_key, crypto_start, part_size, part_number, data_size): + self.crypto_key = crypto_key + self.crypto_start = crypto_start + self.part_size = part_size + self.part_number = part_number + self.data_size = data_size + self.upload_id = None + self.uploaded_parts = set() def _hget(headers, key, converter=lambda x: x): if key in headers: @@ -129,7 +140,7 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi self.__crypto_provider = crypto_provider content_range = _hget(resp.headers, 'Content-Range') - if _hget(resp.headers, 'x-oss-meta-oss-crypto-key') and content_range: + if _hget(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY) and content_range: byte_range = self._parse_range_str(content_range) if not is_multiple_sizeof_encrypt_block(byte_range[0]): raise ClientError('Could not get an encrypted object using byte-range parameter') @@ -143,21 +154,27 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi self.stream = make_crc_adapter(self.stream) if self.__crypto_provider: - key = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, 'x-oss-meta-oss-crypto-key') - count_start = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, 'x-oss-meta-oss-crypto-start') + key = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY) + count_start = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_CRYPTO_START) + key_hmac = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC) + # check the key wrap algorthm is correct + self.__crypto_provider.check_plain_key_valid(key, key_hmac) # if content range , adjust the decrypt adapter count_offset = 0; if content_range: byte_range = self._parse_range_str(content_range) - count_offset = calc_aes_ctr_offset_by_range(byte_range[0]) + count_offset = calc_aes_ctr_offset_by_data_offset(byte_range[0]) - cek_alg = _hget(resp.headers, 'x-oss-meta-oss-cek-alg') + cek_alg = _hget(resp.headers, OSS_CLIENT_SIDE_CRYPTO_CEK_ALG) if key and count_start and cek_alg: self.stream = self.__crypto_provider.make_decrypt_adapter(self.stream, key, count_start, count_offset) else: - raise InconsistentError('all metadata keys are required for decryption (x-oss-meta-oss-crypto-key, \ - x-oss-meta-oss-crypto-start, x-oss-meta-oss-cek-alg)', self.request_id) + err_msg = 'all metadata keys are required for decryption (' \ + + OSS_CLIENT_SIDE_CRYPTO_KEY + ', ' \ + + OSS_CLIENT_SIDE_CRYPTO_START + ', ' \ + + OSS_CLIENT_SIDE_CRYPTO_CEK_ALG + ')' + raise InconsistentError(err_msg, self.request_id) def _parse_range_str(self, content_range): # :param str content_range: sample 'bytes 0-128/1024' @@ -233,6 +250,9 @@ def __init__(self, resp): #: 新生成的Upload ID self.upload_id = None + #: Used in Crypto Bucket + self.part_size = None + self.part_number = None class ListObjectsResult(RequestResult): def __init__(self, resp): diff --git a/oss2/resumable.py b/oss2/resumable.py index 4a574417..89e5fc2d 100644 --- a/oss2/resumable.py +++ b/oss2/resumable.py @@ -19,6 +19,7 @@ from .compat import json, stringify, to_unicode, to_string from .task_queue import TaskQueue from .headers import * +from .utils import _MAX_PART_COUNT, _MIN_PART_SIZE import functools import threading @@ -29,10 +30,6 @@ logger = logging.getLogger(__name__) -_MAX_PART_COUNT = 10000 -_MIN_PART_SIZE = 100 * 1024 - - def resumable_upload(bucket, key, filename, store=None, headers=None, diff --git a/oss2/utils.py b/oss2/utils.py index f8aec5ba..00885a84 100644 --- a/oss2/utils.py +++ b/oss2/utils.py @@ -594,15 +594,40 @@ def random_counter(begin=1, end=10): _AES_GCM = 'AES/GCM/NoPadding' -def is_multiple_sizeof_encrypt_block(byte_range_start): - if byte_range_start is None: - byte_range_start = 0 - return (byte_range_start % _AES_CTR_COUNTER_LEN == 0) - -def calc_aes_ctr_offset_by_range(byte_range_start): - if not is_multiple_sizeof_encrypt_block(byte_range_start): - raise ClientError('Invalid get range value for client encryption') - return byte_range_start / _AES_CTR_COUNTER_LEN +_MAX_PART_COUNT = 10000 +_MIN_PART_SIZE = 100 * 1024 + +def is_multiple_sizeof_encrypt_block(data_offset): + if data_offset is None: + data_offset = 0 + return (data_offset % _AES_CTR_COUNTER_LEN == 0) + +def calc_aes_ctr_offset_by_data_offset(data_offset): + if not is_multiple_sizeof_encrypt_block(data_offset): + raise ClientError('data_offset is not align to encrypt block') + return data_offset / _AES_CTR_COUNTER_LEN + +def is_valid_crypto_part_size(part_size, data_size): + if not is_multiple_sizeof_encrypt_block(part_size) or part_size < _MIN_PART_SIZE: + return False + part_num = (data_size - 1) / part_size + 1 + if part_num > _MAX_PART_COUNT: + return False + return True + +def determine_crypto_part_size(data_size): + if data_size % _MAX_PART_COUNT == 0: + part_size = data_size / _MAX_PART_COUNT + else: + part_size = data_size / (_MAX_PART_COUNT - 1) + + if part_size < _MIN_PART_SIZE: + part_size = _MIN_PART_SIZE + elif not is_multiple_sizeof_encrypt_block(part_size): + part_size = (part_size / _AES_CTR_COUNTER_LEN + 1) * _AES_CTR_COUNTER_LEN + + return part_size + class AESCipher: """AES256 加密实现。 diff --git a/unittests/test_utils.py b/unittests/test_utils.py index ff0a5bac..28a961a7 100644 --- a/unittests/test_utils.py +++ b/unittests/test_utils.py @@ -14,7 +14,7 @@ def test_is_multiple_sizeof_encrypt_block(self): is_multiple = is_multiple_sizeof_encrypt_block(byte_range_start) self.assertFalse(is_multiple) - def test_calc_aes_ctr_offset_by_range(self): + def test_calc_aes_ctr_offset_by_data_offset(self): byte_range_start = 1024 - cout_offset = calc_aes_ctr_offset_by_range(byte_range_start) + cout_offset = calc_aes_ctr_offset_by_data_offset(byte_range_start) self.assertEqual(cout_offset, 1024 / 16) From 5e9b27eebaaf4b7ff21362a294ac4de5d7effb50 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Mon, 8 Oct 2018 17:53:20 +0800 Subject: [PATCH 06/12] add CryptoBucket multipart upload example --- examples/object_crypto.py | 78 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/examples/object_crypto.py b/examples/object_crypto.py index 936d1864..ddf5eff4 100644 --- a/examples/object_crypto.py +++ b/examples/object_crypto.py @@ -5,7 +5,7 @@ import oss2 from oss2.crypto import LocalRsaProvider, AliKMSProvider -# 以下代码展示了客户端文件加密上传下载的用法，如下载文件、上传文件等，注意在客户端加密的条件下，oss暂不支持文件分片上传下载操作。 +# 以下代码展示了客户端文件加密上传下载的用法，如下载文件、上传文件等。 # 首先初始化AccessKeyId、AccessKeySecret、Endpoint等信息。 @@ -31,7 +31,7 @@ filename = 'download.txt' -# 创建Bucket对象，可以进行客户端数据加密(用户端RSA)，此模式下只提供对象整体上传下载操作 +# 创建Bucket对象，可以进行客户端数据加密(用户端RSA) bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, crypto_provider=LocalRsaProvider()) key1 = 'motto-copy.txt' @@ -72,7 +72,43 @@ assert content_got == content[32:1025] -# 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS)，此模式下只提供对象整体上传下载操作 +""" +分片上传 +""" +# 初始化上传分片 +part_a = 'a' * 1024 * 100 +part_b = 'b' * 1024 * 100 +part_c = 'c' * 1024 * 100 +multi_content = [part_a, part_b, part_c] + +parts = [] +data_size = 100 * 1024 * 3 +part_size = 100 * 1024 +multi_key = "test_crypto_multipart" + +res = bucket.init_multipart_upload_securely(multi_key, data_size, part_size) +upload_id = res.upload_id + +# 分片上传 +for i in range(3): + result = bucket.upload_part_securely(multi_key, upload_id, i+1, multi_content[i]) + parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) + +# 完成上传 +result = bucket.complete_multipart_upload_securely(multi_key, upload_id, parts) + +# 下载全部文件 +result = bucket.get_object(multi_key) + +# 验证一下 +content_got = b'' +for chunk in result: + content_got += chunk +assert content_got[0:102400] == part_a +assert content_got[102400:204800] == part_b +assert content_got[204800:307200] == part_c + +# 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS) bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk, '1234')) @@ -112,3 +148,39 @@ for chunk in result: content_got +=chunk assert content_got == content[32:1025] + +""" +分片上传 +""" +# 初始化上传分片 +part_a = 'a' * 1024 * 100 +part_b = 'b' * 1024 * 100 +part_c = 'c' * 1024 * 100 +multi_content = [part_a, part_b, part_c] + +parts = [] +data_size = 100 * 1024 * 3 +part_size = 100 * 1024 +multi_key = "test_crypto_multipart" + +res = bucket.init_multipart_upload_securely(multi_key, data_size, part_size) +upload_id = res.upload_id + +# 分片上传 +for i in range(3): + result = bucket.upload_part_securely(multi_key, upload_id, i+1, multi_content[i]) + parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) + +# 完成上传 +result = bucket.complete_multipart_upload_securely(multi_key, upload_id, parts) + +# 下载全部文件 +result = bucket.get_object(multi_key) + +# 验证一下 +content_got = b'' +for chunk in result: + content_got += chunk +assert content_got[0:102400] == part_a +assert content_got[102400:204800] == part_b +assert content_got[204800:307200] == part_c From 8b56d0d654df299f89bde4b6681074bde7b8f878 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Mon, 8 Oct 2018 20:33:30 +0800 Subject: [PATCH 07/12] add utils unittest --- unittests/test_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/unittests/test_utils.py b/unittests/test_utils.py index 28a961a7..97848ea2 100644 --- a/unittests/test_utils.py +++ b/unittests/test_utils.py @@ -18,3 +18,14 @@ def test_calc_aes_ctr_offset_by_data_offset(self): byte_range_start = 1024 cout_offset = calc_aes_ctr_offset_by_data_offset(byte_range_start) self.assertEqual(cout_offset, 1024 / 16) + + def test_is_valid_crypto_part_size(self): + self.assertFalse(is_valid_crypto_part_size(1, 1024*1024*100)) + self.assertFalse(is_valid_crypto_part_size(1024 * 100 + 1, 1024*1024*100)) + self.assertFalse(is_valid_crypto_part_size(1024 * 100, 1024*1024*1024)) + self.assertTrue(is_valid_crypto_part_size(1024 * 100, 1024*1024*100)) + + def test_determine_crypto_part_size(self): + self.assertEqual(determine_crypto_part_size(1024*100*100000), 1024*1000) + self.assertEqual(determine_crypto_part_size(1024*100*100000 - 1), 1024112) + self.assertEqual(determine_crypto_part_size(1024*100*99), 1024*100) From 651d73c8aa477c6b10ce85c707f7274364ccb6ae Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Tue, 9 Oct 2018 14:05:30 +0800 Subject: [PATCH 08/12] add CryptoBucket multepart upload tests --- examples/object_crypto.py | 12 ++--- oss2/api.py | 7 +-- oss2/models.py | 2 +- oss2/utils.py | 18 ++++++-- tests/test_multipart.py | 96 +++++++++++++++++++++++++++++++++++++++ unittests/test_utils.py | 7 +++ 6 files changed, 126 insertions(+), 16 deletions(-) diff --git a/examples/object_crypto.py b/examples/object_crypto.py index ddf5eff4..afdf95cb 100644 --- a/examples/object_crypto.py +++ b/examples/object_crypto.py @@ -76,9 +76,9 @@ 分片上传 """ # 初始化上传分片 -part_a = 'a' * 1024 * 100 -part_b = 'b' * 1024 * 100 -part_c = 'c' * 1024 * 100 +part_a = b'a' * 1024 * 100 +part_b = b'b' * 1024 * 100 +part_c = b'c' * 1024 * 100 multi_content = [part_a, part_b, part_c] parts = [] @@ -153,9 +153,9 @@ 分片上传 """ # 初始化上传分片 -part_a = 'a' * 1024 * 100 -part_b = 'b' * 1024 * 100 -part_c = 'c' * 1024 * 100 +part_a = b'a' * 1024 * 100 +part_b = b'b' * 1024 * 100 +part_c = b'c' * 1024 * 100 multi_content = [part_a, part_b, part_c] parts = [] diff --git a/oss2/api.py b/oss2/api.py index fe1b269b..728dc4ca 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -1805,7 +1805,7 @@ def init_multipart_upload_securely(self, key, data_size, part_size = None, heade crypto_key = self.crypto_provider.get_key() crypto_start = self.crypto_provider.get_start() - part_number = (data_size - 1) / part_size + 1 + part_number = int((data_size - 1) / part_size + 1) context = CryptoMultipartContext(crypto_key, crypto_start, part_size, part_number, data_size) headers = self.crypto_provider.build_header(headers, context) @@ -1906,7 +1906,7 @@ def abort_multipart_upload_securely(self, key, upload_id): except: raise ClientError("Crypto bucket can't find the upload_id in local contexts") - res = self.bucket.abort_multipart_upload_securely(key, upload_id) + res = self.bucket.abort_multipart_upload(key, upload_id) del self.multipart_upload_contexts[upload_id] logger.info("Abort multipart upload securely done, upload_id = {0} remove from local contexts".format(upload_id)) @@ -1934,9 +1934,6 @@ def list_parts_securely(self, key, upload_id, logger.info("List parts securely done, upload_id = {0}".format(upload_id)) return res - def determine_valid_part_size(data_size): - return determine_crypto_part_size(data_size) - def _normalize_endpoint(endpoint): if not endpoint.startswith('http://') and not endpoint.startswith('https://'): return 'http://' + endpoint diff --git a/oss2/models.py b/oss2/models.py index fd42d441..8780578b 100644 --- a/oss2/models.py +++ b/oss2/models.py @@ -158,7 +158,7 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi count_start = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_CRYPTO_START) key_hmac = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC) # check the key wrap algorthm is correct - self.__crypto_provider.check_plain_key_valid(key, key_hmac) + self.__crypto_provider.check_plain_key_valid(key, to_string(key_hmac)) # if content range , adjust the decrypt adapter count_offset = 0; diff --git a/oss2/utils.py b/oss2/utils.py index 00885a84..4e044ef2 100644 --- a/oss2/utils.py +++ b/oss2/utils.py @@ -615,16 +615,26 @@ def is_valid_crypto_part_size(part_size, data_size): return False return True -def determine_crypto_part_size(data_size): +def determine_crypto_part_size(data_size, excepted_part_size = None): + if excepted_part_size: + # excepted_part_size is valid + if is_valid_crypto_part_size(excepted_part_size, data_size): + return excepted_part_size + # excepted_part_size is enough big but not algin + elif excepted_part_size > data_size/_MAX_PART_COUNT: + part_size = int(excepted_part_size/_AES_CTR_COUNTER_LEN + 1) * _AES_CTR_COUNTER_LEN + return part_size + + # if excepted_part_size is None or is too small, calculate a correct part_size if data_size % _MAX_PART_COUNT == 0: part_size = data_size / _MAX_PART_COUNT else: - part_size = data_size / (_MAX_PART_COUNT - 1) + part_size = int(data_size / (_MAX_PART_COUNT - 1)) if part_size < _MIN_PART_SIZE: part_size = _MIN_PART_SIZE elif not is_multiple_sizeof_encrypt_block(part_size): - part_size = (part_size / _AES_CTR_COUNTER_LEN + 1) * _AES_CTR_COUNTER_LEN + part_size = int(part_size / _AES_CTR_COUNTER_LEN + 1) * _AES_CTR_COUNTER_LEN return part_size @@ -651,7 +661,7 @@ def get_start(): def __init__(self, key=None, count_start=None, count_offset=0): self.key = key - self.count_offset = count_offset + self.count_offset = int(count_offset) if not self.key: self.key = random_aes256_key() if not count_start: diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 3bf840d0..5114c74b 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -97,6 +97,102 @@ def test_upload_part_copy(self): self.assertEqual(len(content_got), len(content)) self.assertEqual(content_got, content) + def do_crypto_multipart_internal(self, do_md5, bucket): + key = self.random_key() + content = random_bytes(100 * 1024) + + parts = [] + data_size = 1024 * 100 + part_size = 1024 * 100 + + res = bucket.init_multipart_upload_securely(key, data_size, part_size) + upload_id = res.upload_id + + multipart_contexts_len = len(bucket.multipart_upload_contexts) + self.assertEqual(multipart_contexts_len, 1) + + if do_md5: + headers = {'Content-Md5': oss2.utils.content_md5(content)} + else: + headers = None + + result = bucket.upload_part_securely(key, upload_id, 1, content) + parts.append(oss2.models.PartInfo(1, result.etag, size = part_size, part_crc = result.crc)) + self.assertTrue(result.crc is not None) + + context_uploaded_parts_len = len(bucket.multipart_upload_contexts[upload_id].uploaded_parts) + self.assertEqual(context_uploaded_parts_len, 1) + + complete_result = bucket.complete_multipart_upload_securely(key, upload_id, parts) + + multipart_contexts_len = len(bucket.multipart_upload_contexts) + self.assertEqual(multipart_contexts_len, 0) + + object_crc = calc_obj_crc_from_parts(parts) + self.assertTrue(complete_result.crc is not None) + self.assertEqual(object_crc, result.crc) + + result = bucket.get_object(key) + self.assertEqual(content, result.read()) + + def do_crypto_abort_internal(self, bucket): + key = self.random_key() + content = random_bytes(100 * 1024) + + data_size = 1024 * 100 + part_size = 1024 * 100 + + res = bucket.init_multipart_upload_securely(key, data_size, part_size) + upload_id = res.upload_id + + result = bucket.upload_part_securely(key, upload_id, 1, content) + + bucket.abort_multipart_upload_securely(key, upload_id) + + multipart_contexts_len = len(bucket.multipart_upload_contexts) + self.assertEqual(multipart_contexts_len, 0) + + def do_crypto_list_parts_internal(self, bucket): + key = self.random_key() + content = random_bytes(100 * 1024) + + data_size = 1024 * 100 + part_size = 1024 * 100 + + res = bucket.init_multipart_upload_securely(key, data_size, part_size) + upload_id = res.upload_id + + bucket.upload_part_securely(key, upload_id, 1, content) + + res = bucket.list_parts_securely(key, upload_id) + self.assertEqual(len(res.parts), 1) + self.assertEqual(res.parts[0].part_number, 1) + + bucket.abort_multipart_upload_securely(key, upload_id) + + def test_rsa_crypto_multipart(self): + self.do_crypto_multipart_internal(False, self.rsa_crypto_bucket) + + def test_rsa_crypto_upload_part_content_md5_good(self): + self.do_crypto_multipart_internal(True, self.rsa_crypto_bucket) + + def test_rsa_crypto_abort(self): + self.do_crypto_abort_internal(self.rsa_crypto_bucket) + + def test_rsa_crypto_list_parts(self): + self.do_crypto_list_parts_internal(self.rsa_crypto_bucket) + + def test_kms_crypto_multipart(self): + self.do_crypto_multipart_internal(False, self.kms_crypto_bucket) + + def test_kms_crypto_upload_part_content_md5_good(self): + self.do_crypto_multipart_internal(True, self.kms_crypto_bucket) + + def test_kms_crypto_abort(self): + self.do_crypto_abort_internal(self.kms_crypto_bucket) + + def test_kms_crypto_list_parts(self): + self.do_crypto_list_parts_internal(self.kms_crypto_bucket) if __name__ == '__main__': unittest.main() diff --git a/unittests/test_utils.py b/unittests/test_utils.py index 97848ea2..e92b0efc 100644 --- a/unittests/test_utils.py +++ b/unittests/test_utils.py @@ -29,3 +29,10 @@ def test_determine_crypto_part_size(self): self.assertEqual(determine_crypto_part_size(1024*100*100000), 1024*1000) self.assertEqual(determine_crypto_part_size(1024*100*100000 - 1), 1024112) self.assertEqual(determine_crypto_part_size(1024*100*99), 1024*100) + + self.assertEqual(determine_crypto_part_size(1024*100*1000, 1024*100), 1024*100) + self.assertEqual(determine_crypto_part_size(1024*100*1000, 1024*100-1), 1024*100) + self.assertEqual(determine_crypto_part_size(1024*100*10000, 1024), 1024*100) + +if __name__ == '__main__': + unittest.main() From 87b6f4b0d8e0c7a50872d8929d8ff85854335e27 Mon Sep 17 00:00:00 2001 From: "wanyuan.ywy" Date: Sat, 2 Feb 2019 22:23:55 +0800 Subject: [PATCH 09/12] add some crypto bucket tests --- examples/object_crypto.py | 8 +- oss2/api.py | 23 +++++- tests/test_object.py | 161 +++++++++++++++++++++++++++++++++++++- 3 files changed, 182 insertions(+), 10 deletions(-) diff --git a/examples/object_crypto.py b/examples/object_crypto.py index afdf95cb..243152ef 100644 --- a/examples/object_crypto.py +++ b/examples/object_crypto.py @@ -19,7 +19,8 @@ access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '<你的AccessKeySecret>') bucket_name = os.getenv('OSS_TEST_BUCKET', '<你的Bucket>') endpoint = os.getenv('OSS_TEST_ENDPOINT', '<你的访问域名>') -cmk = os.getenv('OSS_TEST_CMK', '<你的CMK>') +cmk = os.getenv('OSS_TEST_CMK', '<你的CMK账号>') +cmk_key_secret = os.getenv('OSS_TEST_CMK_KEY_SECRET', '<你的CMK密码>') region = os.getenv('OSS_TEST_REGION', '<你的区域>') # 确认上面的参数都填写正确了 @@ -34,7 +35,6 @@ # 创建Bucket对象，可以进行客户端数据加密(用户端RSA) bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, crypto_provider=LocalRsaProvider()) -key1 = 'motto-copy.txt' # 上传文件 bucket.put_object(key, content, headers={'content-length': str(1024 * 1024)}) @@ -110,9 +110,7 @@ # 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS) bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, - crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk, '1234')) - -key1 = 'motto-copy.txt' + crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk, passphrase = cmk_key_secret)) # 上传文件 bucket.put_object(key, content, headers={'content-length': str(1024 * 1024)}) diff --git a/oss2/api.py b/oss2/api.py index 728dc4ca..2434f9ad 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -623,6 +623,8 @@ def get_object(self, key, resp = self.__do_object('GET', key, headers=headers, params=params) logger.debug("Get object done, req_id: {0}, status_code: {1}".format(resp.request_id, resp.status)) + if models._hget(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY): + raise ClientError('Could not use normal bucket to decrypt an encrypted object') return GetObjectResult(resp, progress_callback, self.enable_crc) def select_object(self, key, sql, @@ -1601,7 +1603,7 @@ def __convert_data(self, klass, converter, data): return data -class CryptoBucket(): +class CryptoBucket(_Base): """用于加密Bucket和Object操作的类，诸如上传、下载Object等。创建、删除bucket的操作需使用Bucket类接口。 用法（假设Bucket属于杭州区域） :: @@ -1643,6 +1645,11 @@ def __init__(self, auth, endpoint, bucket_name, crypto_provider, if not isinstance(crypto_provider, BaseCryptoProvider): raise ClientError('Crypto bucket must provide a valid crypto_provider') + logger.debug("Init oss crypto bucket, endpoint: {0}, isCname: {1}, connect_timeout: {2}, app_name: {3}, enabled_crc: " + "{4}".format(endpoint, is_cname, connect_timeout, app_name, enable_crc)) + super(CryptoBucket, self).__init__(auth, endpoint, is_cname, session, connect_timeout, + app_name, enable_crc) + self.crypto_provider = crypto_provider self.bucket_name = bucket_name.strip() self.enable_crc = enable_crc @@ -1742,9 +1749,16 @@ def get_object(self, key, if range_string: headers['range'] = range_string - encrypted_result = self.bucket.get_object(key, headers=headers, params=params, progress_callback=None) + params = {} if params is None else params + + logger.debug("Start to get object, bucket: {0}， key: {1}, range: {2}, headers: {3}, params: {4}".format( + self.bucket_name, to_string(key), range_string, headers, params)) + resp = self.__do_object('GET', key, headers=headers, params=params) + logger.debug("Get object done, req_id: {0}, status_code: {1}".format(resp.request_id, resp.status)) - return GetObjectResult(encrypted_result.resp, progress_callback, self.enable_crc, + if models._hget(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY) is None: + raise ClientError('Could not use crypto bucket to decrypt an unencrypted object') + return GetObjectResult(resp, progress_callback, self.enable_crc, crypto_provider=self.crypto_provider) def get_object_to_file(self, key, filename, @@ -1934,6 +1948,9 @@ def list_parts_securely(self, key, upload_id, logger.info("List parts securely done, upload_id = {0}".format(upload_id)) return res + def __do_object(self, method, key, **kwargs): + return self._do(method, self.bucket_name, key, **kwargs) + def _normalize_endpoint(endpoint): if not endpoint.startswith('http://') and not endpoint.startswith('https://'): return 'http://' + endpoint diff --git a/tests/test_object.py b/tests/test_object.py index 08dd214a..23477532 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -105,6 +105,77 @@ def test_rsa_crypto_range_get(self): self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(31, None)) self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(None, 31)) + def test_rsa_crypto_object_decrypt_by_normal_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.rsa_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + self.assertRaises(ClientError, self.bucket.get_object, key) + + def test_get_unencrypt_object_decrypt_by_rsa_crypto_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + self.assertRaises(ClientError, self.rsa_crypto_bucket.get_object, key) + + def test_copy_rsa_crypto_object_by_normal_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.rsa_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + copy_key = key + "_copy"; + result = self.bucket.copy_object(self.bucket.bucket_name, key, copy_key) + self.assertTrue(result.status == 200) + + def assert_result(result): + self.assertEqual(result.content_length, len(content)) + self.assertEqual(result.content_type, 'application/javascript') + self.assertEqual(result.object_type, 'Normal') + self.assertTrue(result.etag) + + get_result = self.rsa_crypto_bucket.get_object(copy_key) + self.assertEqual(get_result.read(), content) + assert_result(get_result) + self.assertTrue(get_result.client_crc is not None) + self.assertTrue(get_result.server_crc is not None) + self.assertTrue(get_result.client_crc == get_result.server_crc) + + def test_putlink_rsa_crypto_object_by_normal_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.rsa_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + symlink_key = key + "_symlink"; + result = self.bucket.put_symlink(key, symlink_key) + self.assertTrue(result.status == 200) + + def assert_result(result): + self.assertEqual(result.content_length, len(content)) + self.assertEqual(result.object_type, 'Symlink') + self.assertTrue(result.etag) + + get_result = self.rsa_crypto_bucket.get_object(symlink_key) + self.assertEqual(get_result.read(), content) + assert_result(get_result) + self.assertTrue(get_result.client_crc is not None) + def test_kms_crypto_object(self): if is_py33: return @@ -159,8 +230,94 @@ def test_kms_crypto_range_get(self): get_result = self.kms_crypto_bucket.get_object(key, byte_range=(32, 103)) self.assertEqual(get_result.read(), content[32:103+1]) - self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(31, None)) - self.assertRaises(oss2.exceptions.ClientError, self.rsa_crypto_bucket.get_object, key, byte_range=(None, 31)) + self.assertRaises(oss2.exceptions.ClientError, self.kms_crypto_bucket.get_object, key, byte_range=(31, None)) + self.assertRaises(oss2.exceptions.ClientError, self.kms_crypto_bucket.get_object, key, byte_range=(None, 31)) + + def test_kms_crypto_object_decrypt_by_normal_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.kms_crypto_bucket.put_object(key, content, headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))}) + self.assertTrue(result.status == 200) + + self.assertRaises(ClientError, self.bucket.get_object, key) + + def test_get_unencrypt_object_decrypt_by_kms_crypto_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + self.assertRaises(ClientError, self.kms_crypto_bucket.get_object, key) + + def test_copy_kms_crypto_object_by_normal_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.kms_crypto_bucket.put_object(key, content, headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))}) + self.assertTrue(result.status == 200) + + copy_key = key + "_copy"; + result = self.bucket.copy_object(self.bucket.bucket_name, key, copy_key) + self.assertTrue(result.status == 200) + + def assert_result(result): + self.assertEqual(result.content_length, len(content)) + self.assertEqual(result.content_type, 'application/javascript') + self.assertEqual(result.object_type, 'Normal') + self.assertTrue(result.etag) + + get_result = self.kms_crypto_bucket.get_object(copy_key) + self.assertEqual(get_result.read(), content) + assert_result(get_result) + self.assertTrue(get_result.client_crc is not None) + self.assertTrue(get_result.server_crc is not None) + self.assertTrue(get_result.client_crc == get_result.server_crc) + + def test_putlink_kms_crypto_object_by_normal_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.kms_crypto_bucket.put_object(key, content, headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))}) + self.assertTrue(result.status == 200) + + symlink_key = key + "_symlink"; + result = self.bucket.put_symlink(key, symlink_key) + self.assertTrue(result.status == 200) + + def assert_result(result): + self.assertEqual(result.content_length, len(content)) + self.assertEqual(result.object_type, 'Symlink') + self.assertTrue(result.etag) + + get_result = self.kms_crypto_bucket.get_object(symlink_key) + self.assertEqual(get_result.read(), content) + assert_result(get_result) + self.assertTrue(get_result.client_crc is not None) def test_restore_object(self): auth = oss2.Auth(OSS_ID, OSS_SECRET) From 0df71c2dae34bf7a8f6302d43b7a3cc07b816eb6 Mon Sep 17 00:00:00 2001 From: "wanyuan.ywy" Date: Sun, 3 Feb 2019 11:22:14 +0800 Subject: [PATCH 10/12] fix kms_crypto_bucket example bug --- examples/object_crypto.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/object_crypto.py b/examples/object_crypto.py index 243152ef..d1c3f7e6 100644 --- a/examples/object_crypto.py +++ b/examples/object_crypto.py @@ -19,8 +19,7 @@ access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '<你的AccessKeySecret>') bucket_name = os.getenv('OSS_TEST_BUCKET', '<你的Bucket>') endpoint = os.getenv('OSS_TEST_ENDPOINT', '<你的访问域名>') -cmk = os.getenv('OSS_TEST_CMK', '<你的CMK账号>') -cmk_key_secret = os.getenv('OSS_TEST_CMK_KEY_SECRET', '<你的CMK密码>') +cmk = os.getenv('OSS_TEST_CMK', '<你的CMK>') region = os.getenv('OSS_TEST_REGION', '<你的区域>') # 确认上面的参数都填写正确了 @@ -31,7 +30,6 @@ content = b'a' * 1024 * 1024 filename = 'download.txt' - # 创建Bucket对象，可以进行客户端数据加密(用户端RSA) bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, crypto_provider=LocalRsaProvider()) @@ -110,7 +108,7 @@ # 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS) bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, - crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk, passphrase = cmk_key_secret)) + crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk, '1234')) # 上传文件 bucket.put_object(key, content, headers={'content-length': str(1024 * 1024)}) From 9a22e8ce6ebee2bcaa4bf25b499ea983f69e6489 Mon Sep 17 00:00:00 2001 From: "wanyuan.ywy" Date: Mon, 11 Feb 2019 16:04:33 +0800 Subject: [PATCH 11/12] add upload_id to ClientError msg --- oss2/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oss2/api.py b/oss2/api.py index 2434f9ad..c7b53bc8 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -1853,7 +1853,7 @@ def upload_part_securely(self, key, upload_id, part_number, data, progress_callb try: context = self.multipart_upload_contexts[upload_id] except: - raise ClientError("Crypto bucket can't find the upload_id in local contexts") + raise ClientError("Crypto bucket can't find the upload_id : {0} in local contexts".format(upload_id)) if len(data) != context.part_size and part_number != context.part_number: raise ClientError("Please upload part with correct size unless the last part") From 0c6782b07a856ba6605cf8a8f5d44678257b0306 Mon Sep 17 00:00:00 2001 From: "wanyuan.ywy" Date: Wed, 3 Apr 2019 14:57:49 +0800 Subject: [PATCH 12/12] refine multipart client encryption code --- examples/custom_crypto.py | 95 +++++++- examples/object_crypto.py | 38 ++- oss2/api.py | 99 ++++---- oss2/crypto.py | 92 ++++++-- oss2/exceptions.py | 11 + oss2/headers.py | 34 +-- oss2/models.py | 53 +++-- oss2/xml_utils.py | 14 ++ tests/test_multipart.py | 480 ++++++++++++++++++++++++++++++++++---- tests/test_object.py | 77 ++++-- 10 files changed, 801 insertions(+), 192 deletions(-) diff --git a/examples/custom_crypto.py b/examples/custom_crypto.py index 0aca143d..e7c5d40b 100644 --- a/examples/custom_crypto.py +++ b/examples/custom_crypto.py @@ -5,6 +5,7 @@ import oss2 from oss2.crypto import BaseCryptoProvider from oss2.utils import b64encode_as_string, b64decode_from_string, to_bytes +from oss2.headers import * from Crypto.Cipher import PKCS1_OAEP from Crypto.PublicKey import RSA @@ -36,7 +37,7 @@ def get_key(): def get_start(): return 'fake_start' - def __init__(self, key=None, start=None): + def __init__(self, key=None, start=None, count=None): pass def encrypt(self, raw): @@ -74,22 +75,44 @@ def __init__(self, cipher=FakeCrypto): self.private_key = self.public_key - def build_header(self, headers=None): + def build_header(self, headers=None, multipart_context=None): if not isinstance(headers, CaseInsensitiveDict): headers = CaseInsensitiveDict(headers) if 'content-md5' in headers: - headers['x-oss-meta-unencrypted-content-md5'] = headers['content-md5'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] del headers['content-md5'] if 'content-length' in headers: - headers['x-oss-meta-unencrypted-content-length'] = headers['content-length'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] del headers['content-length'] - headers['x-oss-meta-oss-crypto-key'] = b64encode_as_string(self.public_key.encrypt(self.plain_key)) - headers['x-oss-meta-oss-crypto-start'] = b64encode_as_string(self.public_key.encrypt(to_bytes(str(self.plain_start)))) - headers['x-oss-meta-oss-cek-alg'] = self.cipher.ALGORITHM - headers['x-oss-meta-oss-wrap-alg'] = 'custom' + headers[OSS_CLIENT_SIDE_ENCRYPTION_KEY] = b64encode_as_string(self.public_key.encrypt(self.plain_key)) + headers[OSS_CLIENT_SIDE_ENCRYPTION_START] = b64encode_as_string(self.public_key.encrypt(to_bytes(str(self.plain_start)))) + headers[OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG] = self.cipher.ALGORITHM + headers[OSS_CLIENT_SIDE_ENCRYPTION_WRAP_ALG] = 'custom' + + # multipart file build header + if multipart_context: + headers[OSS_CLIENT_SIDE_ENCRYPTION_DATA_SIZE] = str(multipart_context.data_size) + headers[OSS_CLIENT_SIDE_ENCRYPTION_PART_SIZE] = str(multipart_context.part_size) + + self.plain_key = None + self.plain_start = None + + return headers + + def build_header_for_upload_part(self, headers=None): + if not isinstance(headers, CaseInsensitiveDict): + headers = CaseInsensitiveDict(headers) + + if 'content-md5' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] + del headers['content-md5'] + + if 'content-length' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] + del headers['content-length'] self.plain_key = None self.plain_start = None @@ -110,6 +133,12 @@ def decrypt_oss_meta_data(self, headers, key, conv=lambda x:x): except: return None + def decrypt_from_str(self, key, value, conv=lambda x:x): + try: + return conv(self.private_key.decrypt(b64decode_from_string(value))) + except: + return None + # 首先初始化AccessKeyId、AccessKeySecret、Endpoint等信息。 @@ -162,4 +191,52 @@ def decrypt_oss_meta_data(self, headers, key, conv=lambda x:x): with open(filename, 'rb') as fileobj: assert fileobj.read() == content -os.remove(filename) \ No newline at end of file +os.remove(filename) + +""" +分片上传 +""" +# 初始化上传分片 +part_a = b'a' * 1024 * 100 +part_b = b'b' * 1024 * 100 +part_c = b'c' * 1024 * 100 +multi_content = [part_a, part_b, part_c] + +parts = [] +data_size = 100 * 1024 * 3 +part_size = 100 * 1024 +multi_key = "test_crypto_multipart" + +res = bucket.init_multipart_upload(multi_key, data_size, part_size) +upload_id = res.upload_id +crypto_multipart_context = res.crypto_multipart_context; + +# 分片上传 +for i in range(3): + result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) + parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) + +## 分片上传时，若意外中断丢失crypto_multipart_context, 利用list_parts找回。 +#for i in range(2): +# result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) +# parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) +# +#res = bucket.list_parts(multi_key, upload_id) +#crypto_multipart_context_new = res.crypto_multipart_context +# +#result = bucket.upload_part(multi_key, upload_id, 3, multi_content[2], crypto_multipart_context_new) +#parts.append(oss2.models.PartInfo(3, result.etag, size = part_size, part_crc = result.crc)) + +# 完成上传 +result = bucket.complete_multipart_upload(multi_key, upload_id, parts) + +# 下载全部文件 +result = bucket.get_object(multi_key) + +# 验证一下 +content_got = b'' +for chunk in result: + content_got += chunk +assert content_got[0:102400] == part_a +assert content_got[102400:204800] == part_b +assert content_got[204800:307200] == part_c diff --git a/examples/object_crypto.py b/examples/object_crypto.py index d1c3f7e6..14d5c266 100644 --- a/examples/object_crypto.py +++ b/examples/object_crypto.py @@ -84,16 +84,28 @@ part_size = 100 * 1024 multi_key = "test_crypto_multipart" -res = bucket.init_multipart_upload_securely(multi_key, data_size, part_size) +res = bucket.init_multipart_upload(multi_key, data_size, part_size) upload_id = res.upload_id +crypto_multipart_context = res.crypto_multipart_context; # 分片上传 for i in range(3): - result = bucket.upload_part_securely(multi_key, upload_id, i+1, multi_content[i]) + result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) +## 分片上传时，若意外中断丢失crypto_multipart_context, 利用list_parts找回。 +#for i in range(2): +# result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) +# parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) +# +#res = bucket.list_parts(multi_key, upload_id) +#crypto_multipart_context_new = res.crypto_multipart_context +# +#result = bucket.upload_part(multi_key, upload_id, 3, multi_content[2], crypto_multipart_context_new) +#parts.append(oss2.models.PartInfo(3, result.etag, size = part_size, part_crc = result.crc)) + # 完成上传 -result = bucket.complete_multipart_upload_securely(multi_key, upload_id, parts) +result = bucket.complete_multipart_upload(multi_key, upload_id, parts) # 下载全部文件 result = bucket.get_object(multi_key) @@ -108,7 +120,7 @@ # 创建Bucket对象，可以进行客户端数据加密(使用阿里云KMS) bucket = oss2.CryptoBucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, - crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk, '1234')) + crypto_provider=AliKMSProvider(access_key_id, access_key_secret, region, cmk)) # 上传文件 bucket.put_object(key, content, headers={'content-length': str(1024 * 1024)}) @@ -159,16 +171,28 @@ part_size = 100 * 1024 multi_key = "test_crypto_multipart" -res = bucket.init_multipart_upload_securely(multi_key, data_size, part_size) +res = bucket.init_multipart_upload(multi_key, data_size, part_size) upload_id = res.upload_id +crypto_multipart_context = res.crypto_multipart_context; # 分片上传 for i in range(3): - result = bucket.upload_part_securely(multi_key, upload_id, i+1, multi_content[i]) + result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) +## 分片上传时，若意外中断丢失crypto_multipart_context, 利用list_parts找回。 +#for i in range(2): +# result = bucket.upload_part(multi_key, upload_id, i+1, multi_content[i], crypto_multipart_context) +# parts.append(oss2.models.PartInfo(i+1, result.etag, size = part_size, part_crc = result.crc)) +# +#res = bucket.list_parts(multi_key, upload_id) +#crypto_multipart_context_new = res.crypto_multipart_context +# +#result = bucket.upload_part(multi_key, upload_id, 3, multi_content[2], crypto_multipart_context_new) +#parts.append(oss2.models.PartInfo(3, result.etag, size = part_size, part_crc = result.crc)) + # 完成上传 -result = bucket.complete_multipart_upload_securely(multi_key, upload_id, parts) +result = bucket.complete_multipart_upload(multi_key, upload_id, parts) # 下载全部文件 result = bucket.get_object(multi_key) diff --git a/oss2/api.py b/oss2/api.py index c7b53bc8..160f1e2c 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -623,7 +623,7 @@ def get_object(self, key, resp = self.__do_object('GET', key, headers=headers, params=params) logger.debug("Get object done, req_id: {0}, status_code: {1}".format(resp.request_id, resp.status)) - if models._hget(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY): + if models._hget(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY): raise ClientError('Could not use normal bucket to decrypt an encrypted object') return GetObjectResult(resp, progress_callback, self.enable_crc) @@ -1177,7 +1177,7 @@ def upload_part_copy(self, source_bucket_name, source_key, byte_range, return PutObjectResult(resp) def list_parts(self, key, upload_id, - marker='', max_parts=1000): + marker='', max_parts=1000, headers=None): """列举已经上传的分片。支持分页。 :param str key: 文件名 @@ -1192,7 +1192,8 @@ def list_parts(self, key, upload_id, resp = self.__do_object('GET', key, params={'uploadId': upload_id, 'part-number-marker': marker, - 'max-parts': str(max_parts)}) + 'max-parts': str(max_parts)}, + headers=headers) logger.debug("List parts done, req_id: {0}, status_code: {1}".format(resp.request_id, resp.status)) return self._parse_result(resp, xml_utils.parse_list_parts, ListPartsResult) @@ -1655,7 +1656,6 @@ def __init__(self, auth, endpoint, bucket_name, crypto_provider, self.enable_crc = enable_crc self.bucket = Bucket(auth, endpoint, bucket_name, is_cname, session, connect_timeout, app_name, enable_crc=False) - self.multipart_upload_contexts = {} def put_object(self, key, data, headers=None, @@ -1756,7 +1756,7 @@ def get_object(self, key, resp = self.__do_object('GET', key, headers=headers, params=params) logger.debug("Get object done, req_id: {0}, status_code: {1}".format(resp.request_id, resp.status)) - if models._hget(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY) is None: + if models._hget(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY) is None: raise ClientError('Could not use crypto bucket to decrypt an unencrypted object') return GetObjectResult(resp, progress_callback, self.enable_crc, crypto_provider=self.crypto_provider) @@ -1791,13 +1791,9 @@ def get_object_to_file(self, key, filename, return result - def init_multipart_upload_securely(self, key, data_size, part_size = None, headers=None): + def init_multipart_upload(self, key, data_size, part_size = None, headers=None): """客户端加密初始化分片上传。 - 返回值中的 `upload_id` 以及Bucket名和Object名三元组唯一对应了此次分片上传事件。 - 返回值中的 `part_size` 限制了后续分片上传中除最后一个分片之外其他分片大小必须一致 - 返回值中的 `part_number` 限制了后续分片上传分片总数目，未完全上传不允许complete操作 - :param str key: 待上传的文件名 :param int data_size : 待上传文件总大小 :param int part_size : 后续分片上传时除最后一个分片之外的其他分片大小 @@ -1806,6 +1802,7 @@ def init_multipart_upload_securely(self, key, data_size, part_size = None, heade :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict :return: :class:`InitMultipartUploadResult ` + 返回值中的 `crypto_multipart_context` 记录了加密Meta信息，在upload_part时需要一并传入 """ if part_size is not None: res = is_valid_crypto_part_size(part_size, data_size) @@ -1814,34 +1811,31 @@ def init_multipart_upload_securely(self, key, data_size, part_size = None, heade else: part_size = determine_crypto_part_size(data_size) - logger.info("Start to init multipart upload securely, data_size: {0}, part_size: {1}".format(data_size, part_size)) + logger.info("Start to init multipart upload by crypto bucket, data_size: {0}, part_size: {1}".format(data_size, part_size)) crypto_key = self.crypto_provider.get_key() crypto_start = self.crypto_provider.get_start() part_number = int((data_size - 1) / part_size + 1) - context = CryptoMultipartContext(crypto_key, crypto_start, part_size, part_number, data_size) + context = CryptoMultipartContext(crypto_key, crypto_start, data_size, part_size) headers = self.crypto_provider.build_header(headers, context) resp = self.bucket.init_multipart_upload(key, headers) - resp.part_size = context.part_size - resp.part_number = context.part_number - - context.upload_id = resp.upload_id - self.multipart_upload_contexts[resp.upload_id] = context + resp.crypto_multipart_context = context; - logger.info("Init multipart upload securely done, upload_id = {0} put into local contexts".format(context.upload_id)) + logger.info("Init multipart upload by crypto bucket done, upload_id = {0}.".format(resp.upload_id)) return resp - def upload_part_securely(self, key, upload_id, part_number, data, progress_callback=None, headers=None): + def upload_part(self, key, upload_id, part_number, data, crypto_multipart_context, progress_callback=None, headers=None): """客户端加密上传一个分片。 :param str key: 待上传文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 :param str upload_id: 分片上传ID :param int part_number: 分片号，最小值是1. :param data: 待上传数据。 + :param crypto_multipart_context: 加密Meta信息，在`init_multipart_upload` 时获得 :param progress_callback: 用户指定进度回调函数。可以用来实现进度条等功能。参考 :ref:`progress_callback` 。 :param headers: 用户指定的HTTP头部。可以指定Content-MD5头部等 @@ -1849,18 +1843,15 @@ def upload_part_securely(self, key, upload_id, part_number, data, progress_callb :return: :class:`PutObjectResult ` """ - logger.info("Start upload part securely, upload_id = {0}, part_number = {1}".format(upload_id, part_number)) - try: - context = self.multipart_upload_contexts[upload_id] - except: - raise ClientError("Crypto bucket can't find the upload_id : {0} in local contexts".format(upload_id)) + logger.info("Start upload part by crypto bucket, upload_id = {0}, part_number = {1}".format(upload_id, part_number)) - if len(data) != context.part_size and part_number != context.part_number: - raise ClientError("Please upload part with correct size unless the last part") + headers = http.CaseInsensitiveDict(headers) + headers[FLAG_CLIENT_SIDE_ENCRYPTION_MULTIPART_FILE] = "true" + headers = self.crypto_provider.build_header_for_upload_part(headers) - crypto_key = context.crypto_key - start = context.crypto_start - offset = context.part_size * (part_number - 1) + crypto_key = crypto_multipart_context.crypto_key + start = crypto_multipart_context.crypto_start + offset = crypto_multipart_context.part_size * (part_number - 1) count_offset = utils.calc_aes_ctr_offset_by_data_offset(offset) data = self.crypto_provider.make_encrypt_adapter(data, crypto_key, start, count_offset=count_offset) @@ -1869,13 +1860,12 @@ def upload_part_securely(self, key, upload_id, part_number, data, progress_callb resp = self.bucket.upload_part(key, upload_id, part_number, data, progress_callback, headers) - context.uploaded_parts.add(part_number) - logger.info("Upload part securely done, the part {0} already put into local contexts".format(part_number)) + logger.info("Upload part {0} by crypto bucket done.".format(part_number)) return resp - def complete_multipart_upload_securely(self, key, upload_id, parts, headers=None): + def complete_multipart_upload(self, key, upload_id, parts, headers=None): """客户端加密完成分片上传，创建文件。 当所有分片均已上传成功，才可以调用此函数 @@ -1890,23 +1880,18 @@ def complete_multipart_upload_securely(self, key, upload_id, parts, headers=None :return: :class:`PutObjectResult ` """ - logger.info("Start complete multipart upload securely, upload_id = {0}".format(upload_id)) - try: - context = self.multipart_upload_contexts[upload_id] - except: - raise ClientError("Crypto bucket can't find the upload_id in local contexts") + logger.info("Start complete multipart upload by crypto bucket, upload_id = {0}".format(upload_id)) - if len(context.uploaded_parts) != context.part_number: - raise ClientError("Incomplete parts uploaded in local contexts") + headers = http.CaseInsensitiveDict(headers) + headers[FLAG_CLIENT_SIDE_ENCRYPTION_MULTIPART_FILE] = "true" res = self.bucket.complete_multipart_upload(key, upload_id, parts, headers) - del self.multipart_upload_contexts[upload_id] - logger.info("Complete multipart upload securely done, upload_id = {0} remove from local contexts".format(upload_id)) + logger.info("Complete multipart upload by crypto bucket done, upload_id = {0}.".format(upload_id)) return res - def abort_multipart_upload_securely(self, key, upload_id): + def abort_multipart_upload(self, key, upload_id): """取消分片上传。 :param str key: 待上传的文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 @@ -1914,20 +1899,15 @@ def abort_multipart_upload_securely(self, key, upload_id): :return: :class:`RequestResult ` """ - logger.info("Start abort multipart upload securely, upload_id = {0}".format(upload_id)) - try: - context = self.multipart_upload_contexts[upload_id] - except: - raise ClientError("Crypto bucket can't find the upload_id in local contexts") + logger.info("Start abort multipart upload by crypto bucket, upload_id = {0}".format(upload_id)) res = self.bucket.abort_multipart_upload(key, upload_id) - del self.multipart_upload_contexts[upload_id] - logger.info("Abort multipart upload securely done, upload_id = {0} remove from local contexts".format(upload_id)) + logger.info("Abort multipart upload by crypto bucket done, upload_id = {0}.".format(upload_id)) return res - def list_parts_securely(self, key, upload_id, + def list_parts(self, key, upload_id, marker='', max_parts=1000): """列举已经上传的分片。支持分页。 @@ -1938,14 +1918,19 @@ def list_parts_securely(self, key, upload_id, :return: :class:`ListPartsResult ` """ - logger.info("Start list parts securely, upload_id = {0}".format(upload_id)) - try: - context = self.multipart_upload_contexts[upload_id] - except: - raise ClientError("Crypto bucket can't find the upload_id in local contexts") + logger.info("Start list parts by crypto bucket, upload_id = {0}".format(upload_id)) + + headers = http.CaseInsensitiveDict() + headers[FLAG_CLIENT_SIDE_ENCRYPTION_MULTIPART_FILE] = "true" + + res = self.bucket.list_parts(key, upload_id, marker = marker, max_parts = max_parts, headers=headers) + + crypto_key = self.crypto_provider.decrypt_from_str(OSS_CLIENT_SIDE_ENCRYPTION_KEY, res.crypto_key) + crypto_start = int(self.crypto_provider.decrypt_from_str(OSS_CLIENT_SIDE_ENCRYPTION_START, res.crypto_start)) + context = CryptoMultipartContext(crypto_key, crypto_start, res.client_encryption_data_size, res.client_encryption_part_size) + res.crypto_multipart_context = context - res = self.bucket.list_parts(key, upload_id, marker = marker, max_parts = max_parts) - logger.info("List parts securely done, upload_id = {0}".format(upload_id)) + logger.info("List parts by crypto bucket done, upload_id = {0}".format(upload_id)) return res def __do_object(self, method, key, **kwargs): diff --git a/oss2/crypto.py b/oss2/crypto.py index d4397024..5f8f776c 100644 --- a/oss2/crypto.py +++ b/oss2/crypto.py @@ -96,22 +96,40 @@ def build_header(self, headers=None, multipart_context=None): headers = CaseInsensitiveDict(headers) if 'content-md5' in headers: - headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] del headers['content-md5'] if 'content-length' in headers: - headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] del headers['content-length'] - headers[OSS_CLIENT_SIDE_CRYPTO_KEY] = b64encode_as_string(self.__encrypt_obj.encrypt(self.plain_key)) - headers[OSS_CLIENT_SIDE_CRYPTO_START] = b64encode_as_string(self.__encrypt_obj.encrypt(to_bytes(str(self.plain_start)))) - headers[OSS_CLIENT_SIDE_CRYPTO_CEK_ALG] = self.cipher.ALGORITHM - headers[OSS_CLIENT_SIDE_CRYPTO_WRAP_ALG] = 'rsa' + headers[OSS_CLIENT_SIDE_ENCRYPTION_KEY] = b64encode_as_string(self.__encrypt_obj.encrypt(self.plain_key)) + headers[OSS_CLIENT_SIDE_ENCRYPTION_START] = b64encode_as_string(self.__encrypt_obj.encrypt(to_bytes(str(self.plain_start)))) + headers[OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG] = self.cipher.ALGORITHM + headers[OSS_CLIENT_SIDE_ENCRYPTION_WRAP_ALG] = 'rsa' - headers[OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC] = b64encode_as_string(str(hash(self.plain_key))) + headers[OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC] = b64encode_as_string(str(hash(self.plain_key))) # multipart file build header if multipart_context: - headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH] = str(multipart_context.data_size) + headers[OSS_CLIENT_SIDE_ENCRYPTION_DATA_SIZE] = str(multipart_context.data_size) + headers[OSS_CLIENT_SIDE_ENCRYPTION_PART_SIZE] = str(multipart_context.part_size) + + self.plain_key = None + self.plain_start = None + + return headers + + def build_header_for_upload_part(self, headers=None): + if not isinstance(headers, CaseInsensitiveDict): + headers = CaseInsensitiveDict(headers) + + if 'content-md5' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] + del headers['content-md5'] + + if 'content-length' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] + del headers['content-length'] self.plain_key = None self.plain_start = None @@ -128,13 +146,22 @@ def get_start(self): def decrypt_oss_meta_data(self, headers, key, conv=lambda x:x): try: - if key == OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC: + if key.lower() == OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC.lower(): return conv(utils.b64decode_from_string(headers[key])) else: return conv(self.__decrypt_obj.decrypt(utils.b64decode_from_string(headers[key]))) except: return None + def decrypt_from_str(self, key, value, conv=lambda x:x): + try: + if key.lower() == OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC.lower(): + return conv(utils.b64decode_from_string(value)) + else: + return conv(self.__decrypt_obj.decrypt(utils.b64decode_from_string(value))) + except: + return None + def check_plain_key_valid(self, plain_key, plain_key_hmac): if str(hash(plain_key)) != plain_key_hmac: raise ClientError("The decrypted key is inconsistent, make sure use right RSA key pair") @@ -171,27 +198,45 @@ def build_header(self, headers=None, multipart_context=None): headers = CaseInsensitiveDict(headers) if 'content-md5' in headers: - headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] del headers['content-md5'] if 'content-length' in headers: - headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] del headers['content-length'] - headers[OSS_CLIENT_SIDE_CRYPTO_KEY] = self.encrypted_key - headers[OSS_CLIENT_SIDE_CRYPTO_START] = self.__encrypt_data(to_bytes(str(self.plain_start))) - headers[OSS_CLIENT_SIDE_CRYPTO_CEK_ALG] = self.cipher.ALGORITHM - headers[OSS_CLIENT_SIDE_CRYPTO_WRAP_ALG] = 'kms' + headers[OSS_CLIENT_SIDE_ENCRYPTION_KEY] = self.encrypted_key + headers[OSS_CLIENT_SIDE_ENCRYPTION_START] = self.__encrypt_data(to_bytes(str(self.plain_start))) + headers[OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG] = self.cipher.ALGORITHM + headers[OSS_CLIENT_SIDE_ENCRYPTION_WRAP_ALG] = 'kms' # multipart file build header if multipart_context: - headers[OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH] = str(multipart_context.data_size) + headers[OSS_CLIENT_SIDE_ENCRYPTION_DATA_SIZE] = str(multipart_context.data_size) + headers[OSS_CLIENT_SIDE_ENCRYPTION_PART_SIZE] = str(multipart_context.part_size) self.encrypted_key = None self.plain_start = None return headers + def build_header_for_upload_part(self, headers=None): + if not isinstance(headers, CaseInsensitiveDict): + headers = CaseInsensitiveDict(headers) + + if 'content-md5' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5] = headers['content-md5'] + del headers['content-md5'] + + if 'content-length' in headers: + headers[OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH] = headers['content-length'] + del headers['content-length'] + + self.plain_key = None + self.plain_start = None + + return headers + def get_key(self): plain_key, self.encrypted_key = self.__generate_data_key() return plain_key @@ -260,7 +305,7 @@ def __do(self, req): def decrypt_oss_meta_data(self, headers, key, conv=lambda x: x): try: - if key.lower() == 'x-oss-meta-oss-crypto-key'.lower(): + if key.lower() == OSS_CLIENT_SIDE_ENCRYPTION_KEY.lower(): return conv(b64decode_from_string(self.__decrypt_data(headers[key]))) else: return conv(self.__decrypt_data(headers[key])) @@ -269,6 +314,13 @@ def decrypt_oss_meta_data(self, headers, key, conv=lambda x: x): except: return None - def check_plain_key_valid(self, plain_key, plain_key_hmac): - if plain_key_hmac: - raise ClientError('AliKMSProvider not support check_plain_key_valid') + def decrypt_from_str(self, key, value, conv=lambda x:x): + try: + if key.lower() == OSS_CLIENT_SIDE_ENCRYPTION_KEY.lower(): + return conv(b64decode_from_string(self.__decrypt_data(value))) + else: + return conv(self.__decrypt_data(value)) + except OssError as e: + raise e + except: + return None diff --git a/oss2/exceptions.py b/oss2/exceptions.py index 67518f01..dedcb6f8 100644 --- a/oss2/exceptions.py +++ b/oss2/exceptions.py @@ -146,6 +146,17 @@ class InvalidObjectName(ServerError): status = 400 code = 'InvalidObjectName' +class NotImplemented(ServerError): + status = 400 + code = 'NotImplemented' + +class UnexpectedClientEncryptionPartsList(ServerError): + status = 400 + code = 'UnexpectedClientEncryptionPartsList' + +class DuplicateClientEncryptionMetaSettings(ServerError): + status = 400 + code = 'DuplicateClientEncryptionMetaSettings' class NoSuchBucket(NotFound): status = 404 diff --git a/oss2/headers.py b/oss2/headers.py index 9508f740..86413cc4 100644 --- a/oss2/headers.py +++ b/oss2/headers.py @@ -30,23 +30,25 @@ OSS_SERVER_SIDE_ENCRYPTION = "x-oss-server-side-encryption" OSS_SERVER_SIDE_ENCRYPTION_KEY_ID = "x-oss-server-side-encryption-key-id" -OSS_CLIENT_SIDE_CRYPTO_KEY = "x-oss-meta-oss-crypto-key" -OSS_CLIENT_SIDE_CRYPTO_START = "x-oss-meta-oss-crypto-start" -OSS_CLIENT_SIDE_CRYPTO_CEK_ALG = "x-oss-meta-oss-cek-alg" -OSS_CLIENT_SIDE_CRYPTO_WRAP_ALG = "x-oss-meta-oss-wrap-alg" -OSS_CLIENT_SIDE_CRYTPO_MATDESC = "x-oss-meta-oss-crypto-matdesc" -OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH = "x-oss-meta-oss-crypto-unencrypted-content-length" -OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_MD5 = "x-oss-meta-oss-crypto-unencrypted-content-md5" -OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC = "x-oss-meta-oss-crypto-key-hmac" +OSS_CLIENT_SIDE_ENCRYPTION_KEY = "x-oss-client-side-encryption-key" +OSS_CLIENT_SIDE_ENCRYPTION_START = "x-oss-client-side-encryption-start" +OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG = "x-oss-client-side-encryption-cek-alg" +OSS_CLIENT_SIDE_ENCRYPTION_WRAP_ALG = "x-oss-client-side-encryption-wrap-alg" +OSS_CLIENT_SIDE_ENCRYTPION_MATDESC = "x-oss-client-side-encryption-matdesc" +OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH = "x-oss-client-side-encryption-unencrypted-content-length" +OSS_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5 = "x-oss-client-side-encryption-unencrypted-content-md5" +OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC = "x-oss-client-side-encryption-key-hmac" +OSS_CLIENT_SIDE_ENCRYPTION_DATA_SIZE = "x-oss-client-side-encryption-data-size" +OSS_CLIENT_SIDE_ENCRYPTION_PART_SIZE = "x-oss-client-side-encryption-part-size" +FLAG_CLIENT_SIDE_ENCRYPTION_MULTIPART_FILE = "flag-client-side-encryption-multipart-file" -#OSS_CLIENT_SIDE_CRYPTO_KEY = "x-oss-client-side-crypto-key" -#OSS_CLIENT_SIDE_CRYPTO_START = "x-oss-client-side-crypto-start" -#OSS_CLIENT_SIDE_CRYPTO_CEK_ALG = "x-oss-client-side-crypto-cek-alg" -#OSS_CLIENT_SIDE_CRYPTO_WRAP_ALG = "x-oss-client-side-crypto-wrap-alg" -#OSS_CLIENT_SIDE_CRYTPO_MATDESC = "x-oss-client-side-crypto-matdesc" -#OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_LENGTH = "x-oss-client-side-crypto-unencrypted-content-length" -#OSS_CLIENT_SIDE_CRYPTO_UNENCRYPTED_CONTENT_MD5 = "x-oss-client-side-crypto-unencrypted-content-md5" -#OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC = "x-oss-client-side-crypto-key-hmac" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_KEY = "x-oss-meta-oss-crypto-key" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_START = "x-oss-meta-oss-crypto-start" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_CEK_ALG = "x-oss-meta-oss-cek-alg" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_WRAP_ALG = "x-oss-meta-oss-wrap-alg" +DEPRECATED_CLIENT_SIDE_ENCRYTPION_MATDESC = "x-oss-meta-oss-crypto-matdesc" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_LENGTH = "x-oss-meta-oss-crypto-unencrypted-content-length" +DEPRECATED_CLIENT_SIDE_ENCRYPTION_UNENCRYPTED_CONTENT_MD5 = "x-oss-meta-oss-crypto-unencrypted-content-md5" class RequestHeader(dict): def __init__(self, *arg, **kw): diff --git a/oss2/models.py b/oss2/models.py index 8780578b..393fab74 100644 --- a/oss2/models.py +++ b/oss2/models.py @@ -37,14 +37,11 @@ def __init__(self, part_number, etag, size=None, last_modified=None, part_crc=No class CryptoMultipartContext(object): """表示客户端加密文件通过Multipart接口上传的meta信息 """ - def __init__(self, crypto_key, crypto_start, part_size, part_number, data_size): + def __init__(self, crypto_key, crypto_start, data_size, part_size): self.crypto_key = crypto_key self.crypto_start = crypto_start - self.part_size = part_size - self.part_number = part_number self.data_size = data_size - self.upload_id = None - self.uploaded_parts = set() + self.part_size = part_size def _hget(headers, key, converter=lambda x: x): if key in headers: @@ -140,7 +137,7 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi self.__crypto_provider = crypto_provider content_range = _hget(resp.headers, 'Content-Range') - if _hget(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY) and content_range: + if _hget(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY) and content_range: byte_range = self._parse_range_str(content_range) if not is_multiple_sizeof_encrypt_block(byte_range[0]): raise ClientError('Could not get an encrypted object using byte-range parameter') @@ -154,11 +151,8 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi self.stream = make_crc_adapter(self.stream) if self.__crypto_provider: - key = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY) - count_start = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_CRYPTO_START) - key_hmac = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_CRYPTO_KEY_HMAC) - # check the key wrap algorthm is correct - self.__crypto_provider.check_plain_key_valid(key, to_string(key_hmac)) + key = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY) + count_start = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_START) # if content range , adjust the decrypt adapter count_offset = 0; @@ -166,14 +160,20 @@ def __init__(self, resp, progress_callback=None, crc_enabled=False, crypto_provi byte_range = self._parse_range_str(content_range) count_offset = calc_aes_ctr_offset_by_data_offset(byte_range[0]) - cek_alg = _hget(resp.headers, OSS_CLIENT_SIDE_CRYPTO_CEK_ALG) + cek_alg = _hget(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG) + + # check the key wrap algorthm is correct if rsa + if cek_alg == "rsa": + key_hmac = self.__crypto_provider.decrypt_oss_meta_data(resp.headers, OSS_CLIENT_SIDE_ENCRYPTION_KEY_HMAC) + self.__crypto_provider.check_plain_key_valid(key, to_string(key_hmac)) + if key and count_start and cek_alg: self.stream = self.__crypto_provider.make_decrypt_adapter(self.stream, key, count_start, count_offset) else: err_msg = 'all metadata keys are required for decryption (' \ - + OSS_CLIENT_SIDE_CRYPTO_KEY + ', ' \ - + OSS_CLIENT_SIDE_CRYPTO_START + ', ' \ - + OSS_CLIENT_SIDE_CRYPTO_CEK_ALG + ')' + + OSS_CLIENT_SIDE_ENCRYPTION_KEY + ', ' \ + + OSS_CLIENT_SIDE_ENCRYPTION_START + ', ' \ + + OSS_CLIENT_SIDE_ENCRYPTION_CEK_ALG + ')' raise InconsistentError(err_msg, self.request_id) def _parse_range_str(self, content_range): @@ -250,9 +250,8 @@ def __init__(self, resp): #: 新生成的Upload ID self.upload_id = None - #: Used in Crypto Bucket - self.part_size = None - self.part_number = None + # 客户端加密Bucket关于Multipart文件的context + self.crypto_multipart_context = None class ListObjectsResult(RequestResult): def __init__(self, resp): @@ -271,6 +270,7 @@ def __init__(self, resp): self.prefix_list = [] + class SimplifiedObjectInfo(object): def __init__(self, key, last_modified, etag, type, size, storage_class): #: 文件名，或公共前缀名。 @@ -396,6 +396,23 @@ def __init__(self, resp): # 罗列出的Part信息，类型为 `PartInfo` 列表。 self.parts = [] + # 是否是客户端加密 + self.is_client_encryption = False + + # 客户端加密文件密钥 + self.crypto_key = None + + # 客户端加密文件初始向量 + self.crypto_start = None + + # 客户端加密Multipart文件总大小 + self.client_encryption_data_size = 0 + + # 客户端加密Multipart文件块大小 + self.client_encryption_part_size = 0 + + # 客户端加密Bucket关于Multipart文件的context + self.crypto_multipart_context = None BUCKET_ACL_PRIVATE = 'private' BUCKET_ACL_PUBLIC_READ = 'public-read' diff --git a/oss2/xml_utils.py b/oss2/xml_utils.py index 292e0d8f..f6e4e05f 100644 --- a/oss2/xml_utils.py +++ b/oss2/xml_utils.py @@ -179,6 +179,20 @@ def parse_list_parts(result, body): result.is_truncated = _find_bool(root, 'IsTruncated') result.next_marker = _find_tag(root, 'NextPartNumberMarker') + + try: + result.is_client_encryption = _find_bool(root, 'IsClientEncryption') + result.crypto_key = _find_tag(root, 'ClientEncryptionKey') + result.crypto_start = _find_tag(root, 'ClientEncryptionStart') + result.client_encryption_data_size = _find_int(root, 'ClientEncryptionDataSize') + result.client_encryption_part_size = _find_int(root, 'ClientEncryptionPartSize') + except RuntimeError as e: + result.is_client_encryption = False + result.crypto_key = None + result.crypto_start = None + result.client_encryption_data_size = 0 + result.client_encryption_part_size = 0 + for part_node in root.findall('Part'): result.parts.append(PartInfo( _find_int(part_node, 'PartNumber'), diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 5114c74b..52501f32 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -6,6 +6,7 @@ from common import * +from oss2.compat import is_py2, is_py33 class TestMultipart(OssTestCase): def do_multipart_internal(self, do_md5): @@ -97,102 +98,481 @@ def test_upload_part_copy(self): self.assertEqual(len(content_got), len(content)) self.assertEqual(content_got, content) - def do_crypto_multipart_internal(self, do_md5, bucket): + def do_crypto_multipart_internal(self, do_md5, bucket, is_kms=False): + if is_py33 and is_kms: + return + key = self.random_key() - content = random_bytes(100 * 1024) + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(100 * 1024) + content = [content_1, content_2, content_3] parts = [] + data_size = 1024 * 300 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + self.assertEqual(crypto_multipart_context.data_size, 1024 * 300) + self.assertEqual(crypto_multipart_context.part_size, 1024 * 100) + + for i in range(3): + if do_md5: + headers = {'Content-Md5': oss2.utils.content_md5(content[i])} + else: + headers = None + upload_result = bucket.upload_part(key, upload_id, i+1, content[i], crypto_multipart_context, headers=headers) + parts.append(oss2.models.PartInfo(i+1, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + complete_result = bucket.complete_multipart_upload(key, upload_id, parts) + self.assertTrue(complete_result.status == 200) + + get_result_range_1 = bucket.get_object(key, byte_range=(0, 102399)) + self.assertTrue(get_result_range_1.status == 206) + content_got_1 = get_result_range_1.read() + self.assertEqual(content_1, content_got_1) + + get_result_range_2 = bucket.get_object(key, byte_range=(102400, 204799)) + self.assertTrue(get_result_range_2.status == 206) + content_got_2 = get_result_range_2.read() + self.assertEqual(content_2, content_got_2) + + get_result_range_3 = bucket.get_object(key, byte_range=(204800, 307199)) + self.assertTrue(get_result_range_3.status == 206) + content_got_3 = get_result_range_3.read() + self.assertEqual(content_3, content_got_3) + + get_result = bucket.get_object(key) + self.assertTrue(get_result.status == 200) + content_got = get_result.read() + self.assertEqual(content_1, content_got[0:102400]) + self.assertEqual(content_2, content_got[102400:204800]) + self.assertEqual(content_3, content_got[204800:307200]) + + def do_crypto_abort_multipart(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content = random_bytes(100 * 1024) + data_size = 1024 * 100 part_size = 1024 * 100 - res = bucket.init_multipart_upload_securely(key, data_size, part_size) - upload_id = res.upload_id + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context - multipart_contexts_len = len(bucket.multipart_upload_contexts) - self.assertEqual(multipart_contexts_len, 1) + upload_result = bucket.upload_part(key, upload_id, 1, content, crypto_multipart_context) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) - if do_md5: - headers = {'Content-Md5': oss2.utils.content_md5(content)} - else: - headers = None + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) - result = bucket.upload_part_securely(key, upload_id, 1, content) - parts.append(oss2.models.PartInfo(1, result.etag, size = part_size, part_crc = result.crc)) - self.assertTrue(result.crc is not None) + def do_crypto_list_parts(self, bucket, is_kms=False): + if is_py33 and is_kms: + return - context_uploaded_parts_len = len(bucket.multipart_upload_contexts[upload_id].uploaded_parts) - self.assertEqual(context_uploaded_parts_len, 1) + key = self.random_key() + content = random_bytes(100 * 1024) - complete_result = bucket.complete_multipart_upload_securely(key, upload_id, parts) + data_size = 1024 * 300 + part_size = 1024 * 100 - multipart_contexts_len = len(bucket.multipart_upload_contexts) - self.assertEqual(multipart_contexts_len, 0) + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context - object_crc = calc_obj_crc_from_parts(parts) - self.assertTrue(complete_result.crc is not None) - self.assertEqual(object_crc, result.crc) + upload_result = bucket.upload_part(key, upload_id, 1, content, crypto_multipart_context) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) - result = bucket.get_object(key) - self.assertEqual(content, result.read()) + list_result = bucket.list_parts(key, upload_id) + self.assertTrue(list_result.status == 200) + crypto_multipart_context_new = list_result.crypto_multipart_context + + self.assertEqual(crypto_multipart_context_new.crypto_key, crypto_multipart_context.crypto_key) + self.assertEqual(crypto_multipart_context_new.crypto_start, crypto_multipart_context.crypto_start) + self.assertEqual(crypto_multipart_context_new.data_size, crypto_multipart_context.data_size) + self.assertEqual(crypto_multipart_context_new.part_size, crypto_multipart_context.part_size) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_init_multipart_invalid_parameter(self, bucket, is_kms=False): + if is_py33 and is_kms: + return - def do_crypto_abort_internal(self, bucket): key = self.random_key() content = random_bytes(100 * 1024) data_size = 1024 * 100 + part_size = 1 + + #init multipart with invalid part_size + self.assertRaises(oss2.exceptions.ClientError, bucket.init_multipart_upload, key, data_size, part_size=part_size) + + #init multipart without part_size + init_result = bucket.init_multipart_upload(key, data_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + part_size = crypto_multipart_context.part_size; + self.assertEqual(part_size, 100*1024) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_upload_invalid_part_content(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(50 * 1024) + content = [content_1, content_2, content_3] + content_invalid = random_bytes(100 * 1024 - 1) + + parts = [] + data_size = 1024 * 250 part_size = 1024 * 100 - res = bucket.init_multipart_upload_securely(key, data_size, part_size) - upload_id = res.upload_id + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context - result = bucket.upload_part_securely(key, upload_id, 1, content) + self.assertRaises(oss2.exceptions.InvalidArgument, bucket.upload_part, key, upload_id, 1, content_invalid, crypto_multipart_context) - bucket.abort_multipart_upload_securely(key, upload_id) + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) - multipart_contexts_len = len(bucket.multipart_upload_contexts) - self.assertEqual(multipart_contexts_len, 0) + def do_crypto_upload_invalid_last_part_content(self, bucket, is_kms=False): + if is_py33 and is_kms: + return - def do_crypto_list_parts_internal(self, bucket): key = self.random_key() - content = random_bytes(100 * 1024) + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(50 * 1024) + content = [content_1, content_2, content_3] + content_invalid = random_bytes(100 * 1024 - 1) - data_size = 1024 * 100 + parts = [] + data_size = 1024 * 250 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + for i in range(2): + upload_result = bucket.upload_part(key, upload_id, i+1, content[i], crypto_multipart_context) + parts.append(oss2.models.PartInfo(i+1, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + self.assertRaises(oss2.exceptions.InvalidArgument, bucket.upload_part, key, upload_id, 3, content_invalid, crypto_multipart_context) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_upload_invalid_part_number(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(50 * 1024) + content = [content_1, content_2, content_3] + + parts = [] + data_size = 1024 * 250 part_size = 1024 * 100 - res = bucket.init_multipart_upload_securely(key, data_size, part_size) - upload_id = res.upload_id + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + self.assertRaises(oss2.exceptions.InvalidArgument, bucket.upload_part, key, upload_id, 4, content_1, crypto_multipart_context) + + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_complete_multipart_miss_parts(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(50 * 1024) + content = [content_1, content_2, content_3] + + parts = [] + data_size = 1024 * 250 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + for i in range(2): + upload_result = bucket.upload_part(key, upload_id, i+1, content[i], crypto_multipart_context) + parts.append(oss2.models.PartInfo(i+1, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) - bucket.upload_part_securely(key, upload_id, 1, content) + self.assertRaises(oss2.exceptions.UnexpectedClientEncryptionPartsList, bucket.complete_multipart_upload, key, upload_id, parts) - res = bucket.list_parts_securely(key, upload_id) - self.assertEqual(len(res.parts), 1) - self.assertEqual(res.parts[0].part_number, 1) + abort_result = bucket.abort_multipart_upload(key, upload_id) + self.assertTrue(abort_result.status == 204) - bucket.abort_multipart_upload_securely(key, upload_id) + def do_crypto_resume_upload_after_loss_context(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key = self.random_key() + content_1 = random_bytes(100 * 1024) + content_2 = random_bytes(100 * 1024) + content_3 = random_bytes(100 * 1024) + content = [content_1, content_2, content_3] + + parts = [] + data_size = 1024 * 300 + part_size = 1024 * 100 + + init_result = bucket.init_multipart_upload(key, data_size, part_size) + self.assertTrue(init_result.status == 200) + upload_id = init_result.upload_id + crypto_multipart_context = init_result.crypto_multipart_context + + upload_result = bucket.upload_part(key, upload_id, 1, content[0], crypto_multipart_context) + parts.append(oss2.models.PartInfo(1, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + list_result = bucket.list_parts(key, upload_id) + self.assertTrue(list_result.status == 200) + crypto_multipart_context_new_1 = list_result.crypto_multipart_context + + upload_result = bucket.upload_part(key, upload_id, 2, content[1], crypto_multipart_context_new_1) + parts.append(oss2.models.PartInfo(2, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + list_result = bucket.list_parts(key, upload_id) + self.assertTrue(list_result.status == 200) + crypto_multipart_context_new_2 = list_result.crypto_multipart_context + + upload_result = bucket.upload_part(key, upload_id, 3, content[2], crypto_multipart_context_new_2) + parts.append(oss2.models.PartInfo(3, upload_result.etag, size = part_size, part_crc = upload_result.crc)) + self.assertTrue(upload_result.status == 200) + self.assertTrue(upload_result.crc is not None) + + complete_result = bucket.complete_multipart_upload(key, upload_id, parts) + self.assertTrue(complete_result.status == 200) + + get_result_range_1 = bucket.get_object(key, byte_range=(0, 102399)) + self.assertTrue(get_result_range_1.status == 206) + content_got_1 = get_result_range_1.read() + self.assertEqual(content_1, content_got_1) + + get_result_range_2 = bucket.get_object(key, byte_range=(102400, 204799)) + self.assertTrue(get_result_range_2.status == 206) + content_got_2 = get_result_range_2.read() + self.assertEqual(content_2, content_got_2) + + get_result_range_3 = bucket.get_object(key, byte_range=(204800, 307199)) + self.assertTrue(get_result_range_3.status == 206) + content_got_3 = get_result_range_3.read() + self.assertEqual(content_3, content_got_3) + + get_result = bucket.get_object(key) + self.assertTrue(get_result.status == 200) + content_got = get_result.read() + self.assertEqual(content_1, content_got[0:102400]) + self.assertEqual(content_2, content_got[102400:204800]) + self.assertEqual(content_3, content_got[204800:307200]) + + def do_upload_part_copy_from_crypto_source(self, bucket, crypto_bucket, is_kms=False): + if is_py33 and is_kms: + return + + src_object = self.random_key() + dst_object = self.random_key() + + content = random_bytes(200 * 1024) + + # 上传源文件 + crypto_bucket.put_object(src_object, content) + + # part copy到目标文件 + parts = [] + upload_id = bucket.init_multipart_upload(dst_object).upload_id + + self.assertRaises(oss2.exceptions.NotImplemented, bucket.upload_part_copy, self.bucket.bucket_name, + src_object, (0, 100 * 1024 - 1), dst_object, upload_id, 1) + + abort_result = bucket.abort_multipart_upload(dst_object, upload_id) + self.assertTrue(abort_result.status == 204) + + def do_crypto_multipart_concurrent(self, bucket, is_kms=False): + if is_py33 and is_kms: + return + + key1 = self.random_key() + key1_content_1 = random_bytes(100 * 1024) + key1_content_2 = random_bytes(100 * 1024) + key1_content_3 = random_bytes(100 * 1024) + key1_content = [key1_content_1, key1_content_2, key1_content_3] + + key1_parts = [] + key1_data_size = 1024 * 300 + key1_part_size = 1024 * 100 + + key1_init_result = bucket.init_multipart_upload(key1, key1_data_size, key1_part_size) + self.assertTrue(key1_init_result.status == 200) + key1_upload_id = key1_init_result.upload_id + key1_crypto_multipart_context = key1_init_result.crypto_multipart_context + + self.assertEqual(key1_crypto_multipart_context.data_size, 1024 * 300) + self.assertEqual(key1_crypto_multipart_context.part_size, 1024 * 100) + + key2 = self.random_key() + key2_content_1 = random_bytes(200 * 1024) + key2_content_2 = random_bytes(200 * 1024) + key2_content_3 = random_bytes(100 * 1024) + key2_content = [key2_content_1, key2_content_2, key2_content_3] + + key2_parts = [] + key2_data_size = 1024 * 500 + key2_part_size = 1024 * 200 + + key2_init_result = bucket.init_multipart_upload(key2, key2_data_size, key2_part_size) + self.assertTrue(key2_init_result.status == 200) + key2_upload_id = key2_init_result.upload_id + key2_crypto_multipart_context = key2_init_result.crypto_multipart_context + + self.assertEqual(key2_crypto_multipart_context.data_size, 1024 * 500) + self.assertEqual(key2_crypto_multipart_context.part_size, 1024 * 200) + + for i in range(3): + key1_upload_result = bucket.upload_part(key1, key1_upload_id, i+1, key1_content[i], key1_crypto_multipart_context) + key1_parts.append(oss2.models.PartInfo(i+1, key1_upload_result.etag, size = key1_part_size, part_crc = key1_upload_result.crc)) + self.assertTrue(key1_upload_result.status == 200) + self.assertTrue(key1_upload_result.crc is not None) + + key2_upload_result = bucket.upload_part(key2, key2_upload_id, i+1, key2_content[i], key2_crypto_multipart_context) + key2_parts.append(oss2.models.PartInfo(i+1, key2_upload_result.etag, size = key2_part_size, part_crc = key2_upload_result.crc)) + self.assertTrue(key2_upload_result.status == 200) + self.assertTrue(key2_upload_result.crc is not None) + + key1_complete_result = bucket.complete_multipart_upload(key1, key1_upload_id, key1_parts) + self.assertTrue(key1_complete_result.status == 200) + + key1_get_result = bucket.get_object(key1) + self.assertTrue(key1_get_result.status == 200) + key1_content_got = key1_get_result.read() + self.assertEqual(key1_content_1, key1_content_got[0:102400]) + self.assertEqual(key1_content_2, key1_content_got[102400:204800]) + self.assertEqual(key1_content_3, key1_content_got[204800:307200]) + + key2_complete_result = bucket.complete_multipart_upload(key2, key2_upload_id, key2_parts) + self.assertTrue(key2_complete_result.status == 200) + + key2_get_result = bucket.get_object(key2) + self.assertTrue(key2_get_result.status == 200) + key2_content_got = key2_get_result.read() + self.assertEqual(key2_content_1, key2_content_got[0:204800]) + self.assertEqual(key2_content_2, key2_content_got[204800:409600]) + self.assertEqual(key2_content_3, key2_content_got[409600:512000]) def test_rsa_crypto_multipart(self): - self.do_crypto_multipart_internal(False, self.rsa_crypto_bucket) + self.do_crypto_multipart_internal(False, self.rsa_crypto_bucket, is_kms=False) def test_rsa_crypto_upload_part_content_md5_good(self): - self.do_crypto_multipart_internal(True, self.rsa_crypto_bucket) + self.do_crypto_multipart_internal(True, self.rsa_crypto_bucket, is_kms=False) - def test_rsa_crypto_abort(self): - self.do_crypto_abort_internal(self.rsa_crypto_bucket) + def test_rsa_crypto_abort_multipart(self): + self.do_crypto_abort_multipart(self.rsa_crypto_bucket, is_kms=False) def test_rsa_crypto_list_parts(self): - self.do_crypto_list_parts_internal(self.rsa_crypto_bucket) + self.do_crypto_list_parts(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_init_multipart_invalid_parameter(self): + self.do_crypto_init_multipart_invalid_parameter(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_upload_invalid_part_content(self): + self.do_crypto_upload_invalid_part_content(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_upload_invalid_last_part_content(self): + self.do_crypto_upload_invalid_last_part_content(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_upload_invalid_part_number(self): + self.do_crypto_upload_invalid_part_number(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_complete_multipart_miss_parts(self): + self.do_crypto_complete_multipart_miss_parts(self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_resume_upload_after_loss_context(self): + self.do_crypto_resume_upload_after_loss_context(self.rsa_crypto_bucket, is_kms=False) + + def test_upload_part_copy_from_rsa_crypto_source(self): + self.do_upload_part_copy_from_crypto_source(self.bucket, self.rsa_crypto_bucket, is_kms=False) + + def test_rsa_crypto_multipart_concurrent(self): + self.do_crypto_multipart_concurrent(self.rsa_crypto_bucket, is_kms=False) def test_kms_crypto_multipart(self): - self.do_crypto_multipart_internal(False, self.kms_crypto_bucket) + self.do_crypto_multipart_internal(False, self.kms_crypto_bucket, is_kms=True) def test_kms_crypto_upload_part_content_md5_good(self): - self.do_crypto_multipart_internal(True, self.kms_crypto_bucket) + self.do_crypto_multipart_internal(True, self.kms_crypto_bucket, is_kms=True) - def test_kms_crypto_abort(self): - self.do_crypto_abort_internal(self.kms_crypto_bucket) + def test_kms_crypto_abort_multipart(self): + self.do_crypto_abort_multipart(self.kms_crypto_bucket, is_kms=True) def test_kms_crypto_list_parts(self): - self.do_crypto_list_parts_internal(self.kms_crypto_bucket) + self.do_crypto_list_parts(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_init_multipart_invalid_parameter(self): + self.do_crypto_init_multipart_invalid_parameter(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_upload_invalid_part_content(self): + self.do_crypto_upload_invalid_part_content(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_upload_invalid_last_part_content(self): + self.do_crypto_upload_invalid_last_part_content(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_upload_invalid_part_number(self): + self.do_crypto_upload_invalid_part_number(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_complete_multipart_miss_parts(self): + self.do_crypto_complete_multipart_miss_parts(self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_resume_upload_after_loss_context(self): + self.do_crypto_resume_upload_after_loss_context(self.kms_crypto_bucket, is_kms=True) + + def test_upload_part_copy_from_kms_crypto_source(self): + self.do_upload_part_copy_from_crypto_source(self.bucket, self.kms_crypto_bucket, is_kms=True) + + def test_kms_crypto_multipart_concurrent(self): + self.do_crypto_multipart_concurrent(self.rsa_crypto_bucket, is_kms=True) if __name__ == '__main__': unittest.main() diff --git a/tests/test_object.py b/tests/test_object.py index 23477532..76f5da62 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -133,7 +133,9 @@ def test_copy_rsa_crypto_object_by_normal_bucket(self): self.assertRaises(NotFound, self.bucket.head_object, key) - result = self.rsa_crypto_bucket.put_object(key, content) + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))} + result = self.rsa_crypto_bucket.put_object(key, content, headers=headers) self.assertTrue(result.status == 200) copy_key = key + "_copy"; @@ -153,7 +155,7 @@ def assert_result(result): self.assertTrue(get_result.server_crc is not None) self.assertTrue(get_result.client_crc == get_result.server_crc) - def test_putlink_rsa_crypto_object_by_normal_bucket(self): + def test_replace_rsa_crypto_object_by_normal_bucket(self): key = self.random_key('.js') content = random_bytes(1024) @@ -162,19 +164,40 @@ def test_putlink_rsa_crypto_object_by_normal_bucket(self): result = self.rsa_crypto_bucket.put_object(key, content) self.assertTrue(result.status == 200) - symlink_key = key + "_symlink"; - result = self.bucket.put_symlink(key, symlink_key) + replace_key = key + "_replace"; + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content)), + 'x-oss-metadata-directive':'REPLACE'} + result = self.bucket.copy_object(self.bucket.bucket_name, key, replace_key, headers=headers) self.assertTrue(result.status == 200) def assert_result(result): self.assertEqual(result.content_length, len(content)) - self.assertEqual(result.object_type, 'Symlink') + self.assertEqual(result.content_type, 'application/javascript') + self.assertEqual(result.object_type, 'Normal') self.assertTrue(result.etag) - get_result = self.rsa_crypto_bucket.get_object(symlink_key) + get_result = self.rsa_crypto_bucket.get_object(replace_key) self.assertEqual(get_result.read(), content) assert_result(get_result) self.assertTrue(get_result.client_crc is not None) + self.assertTrue(get_result.server_crc is not None) + self.assertTrue(get_result.client_crc == get_result.server_crc) + + def test_update_crypto_meta_rsa_crypto_object_by_normal_bucket(self): + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.rsa_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content)), + 'x-oss-client-side-encryption-key':'aaaa'} + self.assertRaises(oss2.exceptions.DuplicateClientEncryptionMetaSettings, self.bucket.copy_object, + self.bucket.bucket_name, key, key, headers=headers) def test_kms_crypto_object(self): if is_py33: @@ -271,8 +294,9 @@ def test_copy_kms_crypto_object_by_normal_bucket(self): self.assertRaises(NotFound, self.bucket.head_object, key) - result = self.kms_crypto_bucket.put_object(key, content, headers={'content-md5': oss2.utils.md5_string(content), - 'content-length': str(len(content))}) + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content))} + result = self.kms_crypto_bucket.put_object(key, content, headers=headers) self.assertTrue(result.status == 200) copy_key = key + "_copy"; @@ -292,7 +316,7 @@ def assert_result(result): self.assertTrue(get_result.server_crc is not None) self.assertTrue(get_result.client_crc == get_result.server_crc) - def test_putlink_kms_crypto_object_by_normal_bucket(self): + def test_replace_kms_crypto_object_by_normal_bucket(self): if is_py33: return @@ -301,23 +325,46 @@ def test_putlink_kms_crypto_object_by_normal_bucket(self): self.assertRaises(NotFound, self.bucket.head_object, key) - result = self.kms_crypto_bucket.put_object(key, content, headers={'content-md5': oss2.utils.md5_string(content), - 'content-length': str(len(content))}) + result = self.kms_crypto_bucket.put_object(key, content) self.assertTrue(result.status == 200) - symlink_key = key + "_symlink"; - result = self.bucket.put_symlink(key, symlink_key) + replace_key = key + "_replace"; + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content)), + 'x-oss-metadata-directive':'REPLACE'} + result = self.bucket.copy_object(self.bucket.bucket_name, key, replace_key, headers=headers) self.assertTrue(result.status == 200) def assert_result(result): self.assertEqual(result.content_length, len(content)) - self.assertEqual(result.object_type, 'Symlink') + self.assertEqual(result.content_type, 'application/javascript') + self.assertEqual(result.object_type, 'Normal') self.assertTrue(result.etag) - get_result = self.kms_crypto_bucket.get_object(symlink_key) + get_result = self.kms_crypto_bucket.get_object(replace_key) self.assertEqual(get_result.read(), content) assert_result(get_result) self.assertTrue(get_result.client_crc is not None) + self.assertTrue(get_result.server_crc is not None) + self.assertTrue(get_result.client_crc == get_result.server_crc) + + def test_update_crypto_meta_kms_crypto_object_by_normal_bucket(self): + if is_py33: + return + + key = self.random_key('.js') + content = random_bytes(1024) + + self.assertRaises(NotFound, self.bucket.head_object, key) + + result = self.kms_crypto_bucket.put_object(key, content) + self.assertTrue(result.status == 200) + + headers={'content-md5': oss2.utils.md5_string(content), + 'content-length': str(len(content)), + 'x-oss-client-side-encryption-key':'aaaa'} + self.assertRaises(oss2.exceptions.DuplicateClientEncryptionMetaSettings, self.bucket.copy_object, + self.bucket.bucket_name, key, key, headers=headers) def test_restore_object(self): auth = oss2.Auth(OSS_ID, OSS_SECRET)