From c72ca767cd2f6c6e973cd3274cc83d96ada5f438 Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 24 Jun 2020 19:28:37 +0800 Subject: [PATCH 1/4] modify auth logic --- qcloud_cos/cos_auth.py | 9 ++++----- qcloud_cos/cos_client.py | 13 +++++++++---- qcloud_cos/version.py | 2 +- setup.py | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 3bbacfb6..5c141e94 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -44,14 +44,13 @@ def __call__(self, r): path = self._path uri_params = self._params headers = filter_headers(r.headers) - uri_params = dict([(k.lower(), v) for k, v in uri_params.items()]) # reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+) - headers = dict([(k.lower(), quote(to_bytes(v), '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode - uri_params = dict([(k.lower(), v) for k, v in uri_params.items()]) + headers = dict([(quote(to_bytes(str(k)), '-_.~').lower(), quote(to_bytes(str(v)), '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode + uri_params = dict([(quote(to_bytes(str(k)), '-_.~').lower(), quote(to_bytes(str(v)), '-_.~')) for k, v in uri_params.items()]) format_str = u"{method}\n{host}\n{params}\n{headers}\n".format( method=r.method.lower(), host=path, - params=urlencode(sorted(uri_params.items())).replace('+', '%20').replace('%7E', '~'), + params='&'.join(map(lambda tupl: "%s=%s" % (tupl[0], tupl[1]), sorted(uri_params.items()))), headers='&'.join(map(lambda tupl: "%s=%s" % (tupl[0], tupl[1]), sorted(headers.items()))) ) logger.debug("format str: " + format_str) @@ -73,7 +72,7 @@ def __call__(self, r): ak=self._secret_id, sign_time=sign_time, key_time=sign_time, - params=';'.join(sorted(map(lambda k: k.lower(), uri_params.keys()))), + params=';'.join(sorted(uri_params.keys())), headers=';'.join(sorted(headers.keys())), sign=sign ) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 79d6b0e6..d75e4932 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -2755,7 +2755,7 @@ def list_buckets(self, **kwargs): return data # Advanced interface - def _upload_part(self, bucket, key, local_path, offset, size, part_num, uploadid, md5_lst, resumable_flag, already_exist_parts, enable_md5): + def _upload_part(self, bucket, key, local_path, offset, size, part_num, uploadid, md5_lst, resumable_flag, already_exist_parts, enable_md5, traffic_limit): """从本地文件中读取分块, 上传单个分块,将结果记录在md5——list中 :param bucket(string): 存储桶名称. @@ -2778,12 +2778,13 @@ def _upload_part(self, bucket, key, local_path, offset, size, part_num, uploadid with open(local_path, 'rb') as fp: fp.seek(offset, 0) data = fp.read(size) - rt = self.upload_part(bucket, key, data, part_num, uploadid, enable_md5) + rt = self.upload_part(bucket, key, data, part_num, uploadid, enable_md5, TrafficLimit=traffic_limit) md5_lst.append({'PartNumber': part_num, 'ETag': rt['ETag']}) return None def _get_resumable_uploadid(self, bucket, key): """从服务端获取未完成的分块上传任务,获取断点续传的uploadid + params=urlencode(sorted(uri_params.items())).replace('+', '%20').replace('%7E', '~'), :param bucket(string): 存储桶名称. :param key(string): 分块上传路径名. @@ -2931,15 +2932,19 @@ def upload_file(self, Bucket, Key, LocalFilePath, PartSize=1, MAXThread=5, Enabl logger.info("create a new uploadid in upload_file, uploadid={uploadid}".format(uploadid=uploadid)) # 上传分块 + # 增加限速功能 + traffic_limit = None + if 'TrafficLimit' in kwargs: + traffic_limit = kwargs['TrafficLimit'] offset = 0 # 记录文件偏移量 lst = list() # 记录分块信息 pool = SimpleThreadPool(MAXThread) for i in range(1, parts_num+1): if i == parts_num: # 最后一块 - pool.add_task(self._upload_part, Bucket, Key, LocalFilePath, offset, file_size-offset, i, uploadid, lst, resumable_flag, already_exist_parts, EnableMD5) + pool.add_task(self._upload_part, Bucket, Key, LocalFilePath, offset, file_size-offset, i, uploadid, lst, resumable_flag, already_exist_parts, EnableMD5, traffic_limit) else: - pool.add_task(self._upload_part, Bucket, Key, LocalFilePath, offset, part_size, i, uploadid, lst, resumable_flag, already_exist_parts, EnableMD5) + pool.add_task(self._upload_part, Bucket, Key, LocalFilePath, offset, part_size, i, uploadid, lst, resumable_flag, already_exist_parts, EnableMD5, traffic_limit) offset += part_size pool.wait_completion() diff --git a/qcloud_cos/version.py b/qcloud_cos/version.py index bfa56600..be46c417 100644 --- a/qcloud_cos/version.py +++ b/qcloud_cos/version.py @@ -1,2 +1,2 @@ -__version__ = '5.1.7.8' +__version__ = '5.1.8.1' diff --git a/setup.py b/setup.py index fb32a247..930d2fae 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def long_description(): setup( name='cos-python-sdk-v5', - version='1.7.8', + version='1.8.1', url='https://www.qcloud.com/', license='MIT', author='tiedu, lewzylu, channingliu', From 0d38a1d499a93f1c304b98c3da8e70164b1a08ec Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 24 Jun 2020 19:37:48 +0800 Subject: [PATCH 2/4] modify travis --- .travis.yml | 2 +- qcloud_cos/cos_client.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 92611011..c450a638 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - pip install pycodestyle - pip install dicttoxml script: -- pycodestyle --max-line-length=180 qcloud_cos/. +- pycodestyle --max-line-length=200 qcloud_cos/. - nosetests -s -v ut/ deploy: provider: pypi diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index d75e4932..02ade073 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -2784,7 +2784,6 @@ def _upload_part(self, bucket, key, local_path, offset, size, part_num, uploadid def _get_resumable_uploadid(self, bucket, key): """从服务端获取未完成的分块上传任务,获取断点续传的uploadid - params=urlencode(sorted(uri_params.items())).replace('+', '%20').replace('%7E', '~'), :param bucket(string): 存储桶名称. :param key(string): 分块上传路径名. From 133ba7e459c915de92fbf48781829eee9007e884 Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 24 Jun 2020 20:09:23 +0800 Subject: [PATCH 3/4] py3 compatible --- qcloud_cos/cos_auth.py | 6 +++--- qcloud_cos/cos_comm.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 5c141e94..20287602 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -6,7 +6,7 @@ import hashlib import logging from requests.auth import AuthBase -from .cos_comm import to_unicode, to_bytes +from .cos_comm import to_unicode, to_bytes, to_str logger = logging.getLogger(__name__) @@ -45,8 +45,8 @@ def __call__(self, r): uri_params = self._params headers = filter_headers(r.headers) # reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+) - headers = dict([(quote(to_bytes(str(k)), '-_.~').lower(), quote(to_bytes(str(v)), '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode - uri_params = dict([(quote(to_bytes(str(k)), '-_.~').lower(), quote(to_bytes(str(v)), '-_.~')) for k, v in uri_params.items()]) + headers = dict([(quote(to_bytes(to_str(k)), '-_.~').lower(), quote(to_bytes(to_str(v)), '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode + uri_params = dict([(quote(to_bytes(to_str(k)), '-_.~').lower(), quote(to_bytes(to_str(v)), '-_.~')) for k, v in uri_params.items()]) format_str = u"{method}\n{host}\n{params}\n{headers}\n".format( method=r.method.lower(), host=path, diff --git a/qcloud_cos/cos_comm.py b/qcloud_cos/cos_comm.py index 00403708..8f1e5a90 100644 --- a/qcloud_cos/cos_comm.py +++ b/qcloud_cos/cos_comm.py @@ -61,6 +61,13 @@ } +def to_str(s): + """非字符串转换为字符串""" + if isinstance(s, text_type) or isinstance(s, binary_type): + return s + return str(s) + + def to_unicode(s): """将字符串转为unicode""" if isinstance(s, binary_type): From 29efb60db924ea0d9dae349501fc2e6ba77766d8 Mon Sep 17 00:00:00 2001 From: tiedu Date: Mon, 29 Jun 2020 20:52:36 +0800 Subject: [PATCH 4/4] add ci param support --- qcloud_cos/cos_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 02ade073..67ec16d5 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -405,6 +405,7 @@ def get_presigned_url(self, Bucket, Key, Method, Expired=300, Params={}, Headers """ url = self._conf.uri(bucket=Bucket, path=Key) sign = self.get_auth(Method=Method, Bucket=Bucket, Key=Key, Expired=Expired, Headers=Headers, Params=Params) + sign = urlencode(dict([item.split('=', 1) for item in sign.split('&')])) url = url + '?' + sign if Params: url = url + '&' + urlencode(Params)