From 904e1114bb4d2ebc4ffbd3bf722066ab3962f4d7 Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Mon, 29 Apr 2024 12:44:17 +0800 Subject: [PATCH 01/20] fix ut: test_put_get_delete_website --- ut/test.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/ut/test.py b/ut/test.py index 4bc0f42..c580ec5 100644 --- a/ut/test.py +++ b/ut/test.py @@ -906,6 +906,34 @@ def test_put_get_delete_website(): } ] } + exp_respponse = { + 'IndexDocument': { + 'Suffix': 'index.html' + }, + 'ErrorDocument': { + 'Key': 'error.html' + }, + 'RoutingRules': [ + { + 'Condition': { + 'HttpErrorCodeReturnedEquals': '404', + }, + 'Redirect': { + 'ReplaceKeyWith': '404.html', + 'URLRedirect': 'Enabled' + } + }, + { + 'Condition': { + 'KeyPrefixEquals': 'aaa/' + }, + 'Redirect': { + 'ReplaceKeyPrefixWith': 'ccc/', + 'URLRedirect': 'Enabled' + } + } + ] + } response = client.put_bucket_website( Bucket=test_bucket, WebsiteConfiguration=website_config @@ -916,7 +944,7 @@ def test_put_get_delete_website(): response = client.get_bucket_website( Bucket=test_bucket ) - assert website_config == response + assert exp_respponse == response # delete website response = client.delete_bucket_website( Bucket=test_bucket From a50e234efb99977a74e44755148bd9377a7ac803 Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Mon, 29 Apr 2024 21:14:53 +0800 Subject: [PATCH 02/20] ut: test_select_event_stream_error_message --- ut/test.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/ut/test.py b/ut/test.py index c580ec5..cece758 100644 --- a/ut/test.py +++ b/ut/test.py @@ -11,6 +11,7 @@ from qcloud_cos import CosS3Client from qcloud_cos import CosConfig from qcloud_cos import CosServiceError +from qcloud_cos.select_event_stream import EventStream from qcloud_cos import get_date from qcloud_cos.cos_encryption_client import CosEncryptionClient from qcloud_cos.crypto import AESProvider @@ -1835,6 +1836,80 @@ def test_select_object(): event_stream.get_select_result_to_file(file_name) if os.path.exists(file_name): os.remove(file_name) + + +def test_select_event_stream_error_message(): + ''' + 参考: https://cloud.tencent.com/document/product/436/37641 + 构建EventStream, 测试ErrorMessage处理逻辑 + ''' + import io + import struct + s = io.BytesIO() + header_byte_length = (1+11+1+2+13)+(1+14+1+2+49)+(1+13+1+2+5) + total_byte_length = 4 + 4 + 4 + header_byte_length + 5 + 4 + # Total byte length, Header byte length, Prelude CRC, Headers, Payload, Message CRC + prelude_crc = 1234567890 + + s.write(struct.pack('>I', total_byte_length)) + s.write(struct.pack('>I', header_byte_length)) + s.write(struct.pack('>I', prelude_crc)) + + # Header: error-code + s.write(struct.pack('>B', 11)) # len(':error-code') + s.write(struct.pack('11s', b':error-code')) + s.write(struct.pack('>B', 7)) + s.write(struct.pack('>H', 13)) # len('InternalError') + s.write(struct.pack('13s', b'InternalError')) + + # Header: error-message + s.write(struct.pack('>B', 14)) # len(':error-message') + s.write(struct.pack('14s', b':error-message')) + s.write(struct.pack('>B', 7)) + s.write(struct.pack('>H', 49)) # len('We encounted an internal error. Please try again.') + s.write(struct.pack('49s', b'We encounted an internal error. Please try again.')) + + # Header: message-type + s.write(struct.pack('>B', 13)) # len(':message-type') + s.write(struct.pack('13s', b':message-type')) + s.write(struct.pack('>B', 7)) + s.write(struct.pack('>H', 5)) # len('error') + s.write(struct.pack('5s', b'error')) + + # Payload + s.write(struct.pack('5s', b'AAAAA')) + + # Message CRC + s.write(struct.pack('>I', 1234567890)) + + s.seek(0, 0) + + class request_obj(): + def __init__(self): + self.url = 'http://www.test.com' + + class rt_obj(): + def __init__(self): + self.raw = s + self.request = request_obj() + self.status_code = 400 + self.headers = { + 'x-cos-request-id': 'xxx', + 'x-cos-trace-id': 'yyy', + } + + rt = rt_obj() + event_stream = EventStream(rt) + try: + for ev in event_stream: + print(ev) + except CosServiceError as e: + print(e) + assert e.get_error_code() == 'InternalError' + assert e.get_error_msg() == 'We encounted an internal error. Please try again.' + assert e.get_resource_location() == 'http://www.test.com' + assert e.get_request_id() == 'xxx' + assert e.get_trace_id() == 'yyy' def test_get_object_sensitive_content_recognition(): From 60ab9aa8a8a24d46cf055ef30173c27e9b2669d2 Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Mon, 6 May 2024 10:31:15 +0800 Subject: [PATCH 03/20] rm duplicate statement --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 1ccf4d0..2bfb86c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ from platform import python_version_tuple import sys import io -import sys def requirements(): From 0ebb339c00d322d258eabc69145616f3ce28a1ad Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Mon, 6 May 2024 16:03:11 +0800 Subject: [PATCH 04/20] =?UTF-8?q?=E9=87=8D=E8=AF=95=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E3=80=81UT=E8=A1=A5=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qcloud_cos/cos_client.py | 35 ++++++++++------- ut/test.py | 83 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 15 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 9db7d5b..b998e81 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -331,6 +331,14 @@ def get_auth(self, Method, Bucket, Key, Expired=300, Headers={}, Params={}, Sign auth = CosS3Auth(self._conf, Key, Params, Expired, SignHost) return auth(r).headers['Authorization'] + def should_switch_domain(self, domain_switched, headers={}): + if not 'x-cos-request-id' in headers and \ + not domain_switched and \ + self._conf._auto_switch_domain_on_retry and \ + self._conf._ip is None: + return True + return False + def send_request(self, method, url, bucket, timeout=30, cos_request=True, ci_request=False, **kwargs): """封装request库发起http请求""" if self._conf._timeout is not None: # 用户自定义超时时间 @@ -372,7 +380,10 @@ def send_request(self, method, url, bucket, timeout=30, cos_request=True, ci_req for j in range(self._retry + 1): try: if j != 0: - time.sleep(j) + if client_can_retry(file_position, **kwargs): + time.sleep(j) + else: + break if method == 'POST': res = self._session.post(url, timeout=timeout, proxies=self._conf._proxies, **kwargs) elif method == 'GET': @@ -385,24 +396,21 @@ def send_request(self, method, url, bucket, timeout=30, cos_request=True, ci_req res = self._session.head(url, timeout=timeout, proxies=self._conf._proxies, **kwargs) if res.status_code < 400: # 2xx和3xx都认为是成功的 if res.status_code == 301 or res.status_code == 302 or res.status_code == 307: - if j < self._retry and not 'x-cos-request-id' in res.headers \ - and not domain_switched and self._conf._auto_switch_domain_on_retry and self._conf._ip is None: + if j < self._retry and self.should_switch_domain(domain_switched, res.headers): url = switch_hostname_for_url(url) domain_switched = True continue return res elif res.status_code < 500: # 4xx 不重试 - if j < self._retry and not 'x-cos-request-id' in res.headers \ - and not domain_switched and self._conf._auto_switch_domain_on_retry and self._conf._ip is None: + if j < self._retry and self.should_switch_domain(domain_switched, res.headers): url = switch_hostname_for_url(url) domain_switched = True continue break else: - if j < self._retry and client_can_retry(file_position, **kwargs): - if not 'x-cos-request-id' in res.headers and not domain_switched and self._conf._auto_switch_domain_on_retry and self._conf._ip is None: - url = switch_hostname_for_url(url) - domain_switched = True + if j < self._retry and self.should_switch_domain(domain_switched, res.headers): + url = switch_hostname_for_url(url) + domain_switched = True continue else: break @@ -411,11 +419,10 @@ def send_request(self, method, url, bucket, timeout=30, cos_request=True, ci_req exception_log = 'url:%s, retry_time:%d exception:%s' % (url, j, str(e)) exception_logbuf.append(exception_log) if j < self._retry and (isinstance(e, ConnectionError) or isinstance(e, Timeout)): # 只重试网络错误 - if client_can_retry(file_position, **kwargs): - if not domain_switched and self._conf._auto_switch_domain_on_retry and self._conf._ip is None: - url = switch_hostname_for_url(url) - domain_switched = True - continue + if self.should_switch_domain(domain_switched): + url = switch_hostname_for_url(url) + domain_switched = True + continue logger.exception(exception_logbuf) # 最终重试失败, 输出前几次重试失败的exception raise CosClientError(str(e)) diff --git a/ut/test.py b/ut/test.py index cece758..44c621b 100644 --- a/ut/test.py +++ b/ut/test.py @@ -10,7 +10,7 @@ from qcloud_cos import CosS3Client from qcloud_cos import CosConfig -from qcloud_cos import CosServiceError +from qcloud_cos import CosServiceError, CosClientError from qcloud_cos.select_event_stream import EventStream from qcloud_cos import get_date from qcloud_cos.cos_encryption_client import CosEncryptionClient @@ -3766,6 +3766,87 @@ def test_switch_hostname_for_url(): except Exception as e: print(e) + +def test_should_switch_domain(): + conf1 = CosConfig( + Region=REGION, + SecretId=SECRET_ID, + SecretKey=SECRET_KEY, + ) + client1 = CosS3Client(conf1) + domain_switched = False + headers = {} + # 默认AutoSwitchedDomainOnRetry=False, 不切换域名 + assert client1.should_switch_domain(domain_switched, headers) == False + + conf1 = CosConfig( + Region=REGION, + SecretId=SECRET_ID, + SecretKey=SECRET_KEY, + AutoSwitchDomainOnRetry=True, + ) + client1 = CosS3Client(conf1) + domain_switched = False + headers = {} + # AutoSwitchedDomainOnRetry=True, 切换域名 + assert client1.should_switch_domain(domain_switched, headers) == True + + conf1 = CosConfig( + Region=REGION, + SecretId=SECRET_ID, + SecretKey=SECRET_KEY, + AutoSwitchDomainOnRetry=True, + ) + client1 = CosS3Client(conf1) + domain_switched = True # 已经切换过了, 本次不切换 + headers = {} + assert client1.should_switch_domain(domain_switched, headers) == False + + conf1 = CosConfig( + Region=REGION, + SecretId=SECRET_ID, + SecretKey=SECRET_KEY, + AutoSwitchDomainOnRetry=True, + ) + client1 = CosS3Client(conf1) + domain_switched = True + headers = {'x-cos-request-id': 'xxx'} + # 响应头中有x-cos-request-id, 不切换域名 + assert client1.should_switch_domain(domain_switched, headers) == False + + conf1 = CosConfig( + Region=REGION, + SecretId=SECRET_ID, + SecretKey=SECRET_KEY, + AutoSwitchDomainOnRetry=True, + ) + conf1.set_ip_port('10.0.0.1', 443) + client1 = CosS3Client(conf1) + domain_switched = True + headers = {} + # 请求指定了ip, 不切换域名 + assert client1.should_switch_domain(domain_switched, headers) == False + +def test_network_failure(): + """指定一个错误的ip""" + conf1 = CosConfig( + Region=REGION, + SecretId=SECRET_ID, + SecretKey=SECRET_KEY, + Scheme='http', + Timeout=10, + AutoSwitchDomainOnRetry=True, + ) + conf1.set_ip_port('10.0.0.1', 80) + client1 = CosS3Client(conf1) + try: + response = client1.get_object( + Bucket=test_bucket, + Key='test', + ) + except CosClientError as e: + print(e) + if __name__ == "__main__": setUp() """ From efdb0da08a1ecaa288af7ef3577a8f15669f356d Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Mon, 6 May 2024 16:38:19 +0800 Subject: [PATCH 05/20] fix ut: test_download_file --- ut/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ut/test.py b/ut/test.py index 44c621b..5195ed9 100644 --- a/ut/test.py +++ b/ut/test.py @@ -1936,7 +1936,7 @@ def test_download_file(): os.remove('test_download_file.local') # 重置内置线程池大小 - client.set_built_in_connection_pool_max_size(10, 5) + client.generate_built_in_connection_pool(10, 5) # 测试限速下载 client.download_file(copy_test_bucket, test_object, 'test_download_traffic_limit.local', TrafficLimit='819200') From c7852d876d06ddb797174aceb07d28c1e277276f Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Mon, 6 May 2024 20:33:25 +0800 Subject: [PATCH 06/20] ut --- ut/test.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/ut/test.py b/ut/test.py index 5195ed9..5391142 100644 --- a/ut/test.py +++ b/ut/test.py @@ -7,6 +7,7 @@ import requests import json import base64 +import multiprocessing from qcloud_cos import CosS3Client from qcloud_cos import CosConfig @@ -1102,6 +1103,42 @@ def test_upload_file_multithreading(): os.remove(file_name) print(ed - st) +def multiprocessing_worker(file_name): + gen_file(file_name, 10) + client = CosS3Client(conf) + response = client.upload_file( + Bucket=test_bucket, + Key=file_name, + LocalFilePath=file_name, + PartSize=1, + MAXThread=5 + ) + assert response + if os.path.exists(file_name): + os.remove(file_name) + +def test_upload_file_multiprocessing(): + """多进程+多线程上传10M的文件""" + file_name = 'test_10M' + gen_file(file_name, 10) + # 主进程先做一次请求, 将socket连接保留在连接池里, 子进程应该重新生成自己的连接池 + response = client.upload_file( + Bucket=test_bucket, + Key=file_name, + LocalFilePath=file_name, + PartSize=1, + MAXThread=5 + ) + assert response + if os.path.exists(file_name): + os.remove(file_name) + + pool = multiprocessing.Pool(2) + pool.apply_async(multiprocessing_worker, args=('test1_10M',)) + pool.apply_async(multiprocessing_worker, args=('test2_10M',)) + pool.close() + pool.join() + def test_upload_file_with_progress_callback(): """带有进度条功能的并发上传""" From 700aebad414ba1d1ecf5390a3396e0ef62f013e7 Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Tue, 7 May 2024 13:07:32 +0800 Subject: [PATCH 07/20] ut --- qcloud_cos/cos_client.py | 2 +- ut/test.py | 575 ++++++++++++++++++++++++++------------- 2 files changed, 392 insertions(+), 185 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index b998e81..370fb2c 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -3949,7 +3949,7 @@ def upload_file_from_buffer(self, Bucket, Key, Body, MaxBufferSize=100, PartSize :return(dict): 成功上传的文件的结果. """ if not hasattr(Body, 'read'): - raise CosClientError("Body must has attr read") + raise CosClientError("Body must have attr read") part_size = 1024 * 1024 * PartSize diff --git a/ut/test.py b/ut/test.py index 5391142..f7a1f1d 100644 --- a/ut/test.py +++ b/ut/test.py @@ -30,22 +30,26 @@ sys.version_info[1]) + '-' + REGION + '-' + APPID copy_test_bucket = 'copy-' + test_bucket test_object = "test.txt" -special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" +special_file_name = "中文" + \ + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" """ CredentialDemo """ + + class CredentialDemo: @property def secret_id(self): return SECRET_ID - + @property def secret_key(self): return SECRET_KEY - + @property def token(self): return '' + if USE_CREDENTIAL_INST == 'true': conf = CosConfig( Region=REGION, @@ -99,6 +103,7 @@ def _create_test_bucket(test_bucket, create_region=None): raise e return None + def _clear_and_delete_bucket(_client, _bucket): marker = "" while True: @@ -109,7 +114,7 @@ def _clear_and_delete_bucket(_client, _bucket): for content in response['Contents']: key = content['Key'] objects.append({'Key': key}) - + if response['IsTruncated'] == 'false': break @@ -127,6 +132,7 @@ def _clear_and_delete_bucket(_client, _bucket): ) print("bucket deleted: {}".format(_bucket)) + def _upload_test_file(test_bucket, test_key): response = client.put_object( Bucket=test_bucket, @@ -135,6 +141,7 @@ def _upload_test_file(test_bucket, test_key): ) return None + def _upload_test_file_from_local_file(test_bucket, test_key, file_name): with open(file_name, 'rb') as f: response = client.put_object( @@ -144,23 +151,26 @@ def _upload_test_file_from_local_file(test_bucket, test_key, file_name): ) return None + def get_raw_md5(data): m2 = hashlib.md5(data) etag = '"' + str(m2.hexdigest()) + '"' return etag -def gen_file(path, size): +def gen_file(path, size, attach_size=0): _file = open(path, 'w') - _file.seek(1024 * 1024 * size - 3) + _file.seek(1024 * 1024 * size + attach_size - 3) _file.write('cos') _file.close() + def gen_file_small(path, size): _file = open(path, 'w') _file.write('x'*size) _file.close() + def print_error_msg(e): print(e.get_origin_msg()) print(e.get_digest_msg()) @@ -195,6 +205,7 @@ def setUp(): def tearDown(): print("function teardown") + def test_cos_comm_format_region(): from qcloud_cos.cos_comm import format_region try: @@ -211,7 +222,7 @@ def test_cos_comm_format_region(): r = format_region('ap_beijing', u'cos.', False, False) except Exception as e: print(e) - + r = format_region('cos.ap-beijing', u'cos.', False, False) assert r == 'cos.ap-beijing' @@ -233,7 +244,8 @@ def test_cos_comm_format_region(): r = format_region('sg', u'cos.', False, False) assert r == 'sg' - r = format_region('ap-beijing', u'cos.', EnableOldDomain=False, EnableInternalDomain=True) + r = format_region('ap-beijing', u'cos.', + EnableOldDomain=False, EnableInternalDomain=True) assert r == 'cos-internal.ap-beijing' regionMap = [ @@ -251,6 +263,7 @@ def test_cos_comm_format_region(): r = format_region(data[0], u'cos.', False, False) assert r == 'cos.' + data[1] + def test_cos_comm_format_bucket(): from qcloud_cos.cos_comm import format_bucket @@ -399,7 +412,8 @@ def test_get_object_acl(): def test_copy_object_diff_bucket(): """从另外的bucket拷贝object""" - copy_source = {'Bucket': copy_test_bucket, 'Key': 'test.txt', 'Region': REGION} + copy_source = {'Bucket': copy_test_bucket, + 'Key': 'test.txt', 'Region': REGION} response = client.copy_object( Bucket=test_bucket, Key='test.txt', @@ -491,7 +505,8 @@ def test_upload_part_copy(): ) # upload part copy - copy_source = {'Bucket': copy_test_bucket, 'Key': 'test.txt', 'Region': REGION} + copy_source = {'Bucket': copy_test_bucket, + 'Key': 'test.txt', 'Region': REGION} response = client.upload_part_copy( Bucket=test_bucket, Key='multipartfile.txt', @@ -560,10 +575,12 @@ def test_create_head_delete_bucket(): response = client.head_bucket( Bucket=bucket_name ) + assert response response = client.delete_bucket( Bucket=bucket_name ) + def test_create_head_delete_maz_ofs_bucket(): """创建一个多AZ OFS bucket,head它是否存在,最后删除一个空bucket""" bucket_id = str(random.randint(0, 1000)) + str(random.randint(0, 1000)) @@ -577,6 +594,7 @@ def test_create_head_delete_maz_ofs_bucket(): response = client.head_bucket( Bucket=bucket_name ) + assert response response = client.delete_bucket( Bucket=bucket_name ) @@ -590,6 +608,7 @@ def test_put_bucket_acl(): ) print(response) + def test_put_bucket_acl_illegal(): """设置非法的ACL""" try: @@ -600,6 +619,32 @@ def test_put_bucket_acl_illegal(): except CosServiceError as e: print_error_msg(e) + try: + response = client.put_bucket_acl( + Bucket=test_bucket, + ACL='private', + AccessControlPolicy={ + 'AccessControlList': { + 'Grant': [ + { + 'Grantee': { + 'DisplayName': 'qcs::cam::uin/100000000002:uin/100000000002', + 'Type': 'CanonicalUser', + 'ID': 'qcs::cam::uin/100000000002:uin/100000000002', # Type为CanonicalUser时必须填写ID + }, + 'Permission': 'WRITE' + }, + ] + }, + 'Owner': { + 'DisplayName': 'qcs::cam::uin/100000000001:uin/100000000001', + 'ID': 'qcs::cam::uin/100000000001:uin/100000000001' # 必须是桶 Owner 的 ID + } + } + ) + except CosServiceError as e: + print_error_msg(e) + def test_get_bucket_acl_normal(): """正常获取bucket ACL""" @@ -640,6 +685,7 @@ def test_list_objects(): except Exception as e: print(e) + def test_list_objects_versions(): """列出bucket下的带版本信息的objects""" response = client.list_objects_versions( @@ -709,12 +755,13 @@ def test_get_service(): } } ) - response = client.list_buckets(Region=REGION, TagKey='tagKey', TagValue='tagValue') + response = client.list_buckets( + Region=REGION, TagKey='tagKey', TagValue='tagValue') for bucket in response['Buckets']['Bucket']: tag = client.get_bucket_tagging(Bucket=bucket['Name']) assert tag['TagSet']['Tag'][0]['Key'] == 'tagKey' assert tag['TagSet']['Tag'][0]['Value'] == 'tagValue' - + time.sleep(3) client.delete_bucket(Bucket=test_tagging_bucket) @@ -723,9 +770,11 @@ def test_get_service(): list_over = False while list_over is False: create_time = 1514736000 - response = client.list_buckets(Region='ap-beijing', CreateTime=create_time, Range='gt', Marker=marker) + response = client.list_buckets( + Region='ap-beijing', CreateTime=create_time, Range='gt', Marker=marker) for bucket in response['Buckets']['Bucket']: - ctime = int(time.mktime(datetime.strptime(bucket['CreationDate'], '%Y-%m-%dT%H:%M:%SZ').timetuple())) + ctime = int(time.mktime(datetime.strptime( + bucket['CreationDate'], '%Y-%m-%dT%H:%M:%SZ').timetuple())) assert ctime > create_time assert bucket['Location'] == 'ap-beijing' @@ -999,7 +1048,7 @@ def test_list_multipart_uploads(): Bucket=test_bucket, Prefix="multipart", MaxUploads=100, - EncodingType='xml' # 非法, 只能为url + EncodingType='xml' # 非法, 只能为url ) except Exception as e: print(e) @@ -1024,6 +1073,29 @@ def test_upload_file_from_buffer(): PartSize=1 ) + # 简单上传 + data = io.BytesIO(1024 * b'A') + response = client.upload_file_from_buffer( + Bucket=test_bucket, + Key='test_upload_from_buffer', + Body=data, + MaxBufferSize=5, + PartSize=1 + ) + + # Body没有read方法 + try: + response = client.upload_file_from_buffer( + Bucket=test_bucket, + Key='test_upload_from_buffer', + Body=b'xxx', + MaxBufferSize=5, + PartSize=1 + ) + except CosClientError as e: + print(e) # Body must have attr read + + def test_upload_small_file(): """使用高级上传接口上传小文件""" file_name = "file_1M" @@ -1083,6 +1155,33 @@ def test_upload_small_file(): ) assert response['Content-Length'] == '0' + +def test_upload_file_10000_parts_with_trafficlimit(): + """将文件最大分块数限制在10000""" + file_name = "file_10000_parts" + file_size = 10000 + attach_size = 1024 # 1KB + gen_file(file_name, file_size, attach_size) + + st = time.time() # 记录开始时间 + try: + response = client.upload_file( + Bucket=test_bucket, + Key=file_name, + LocalFilePath=file_name, + MAXThread=10, + PartSize=1, + TrafficLimit=10000000 + ) + except CosClientError as e: + print(e) + + ed = time.time() # 记录结束时间 + if os.path.exists(file_name): + os.remove(file_name) + print(ed - st) + + def test_upload_file_multithreading(): """根据文件大小自动选择分块大小,多线程并发上传提高上传速度""" file_name = "thread_1GB" @@ -1103,6 +1202,7 @@ def test_upload_file_multithreading(): os.remove(file_name) print(ed - st) + def multiprocessing_worker(file_name): gen_file(file_name, 10) client = CosS3Client(conf) @@ -1117,6 +1217,7 @@ def multiprocessing_worker(file_name): if os.path.exists(file_name): os.remove(file_name) + def test_upload_file_multiprocessing(): """多进程+多线程上传10M的文件""" file_name = 'test_10M' @@ -1161,12 +1262,14 @@ def test_upload_file_with_progress_callback(): def test_copy_file_automatically(): """根据拷贝源文件的大小自动选择拷贝策略，不同园区,小于5G直接copy_object，大于5G分块拷贝""" - copy_source = {'Bucket': copy_test_bucket, 'Key': 'test.txt', 'Region': REGION} + copy_source = {'Bucket': copy_test_bucket, + 'Key': 'test.txt', 'Region': REGION} response = client.copy( Bucket=test_bucket, Key='copy.txt', CopySource=copy_source, - MAXThread=10 + MAXThread=10, + StorageClass='STANDARD' ) @@ -1194,9 +1297,11 @@ def test_use_get_auth(): Params={'acl': '', 'unsed': '123'} ) if conf._enable_old_domain: - url = 'http://' + test_bucket + '.cos.' + REGION + '.myqcloud.com/test.txt?acl&unsed=123' + url = 'http://' + test_bucket + '.cos.' + \ + REGION + '.myqcloud.com/test.txt?acl&unsed=123' else: - url = 'http://' + test_bucket + '.cos.' + REGION + '.tencentcos.cn/test.txt?acl&unsed=123' + url = 'http://' + test_bucket + '.cos.' + REGION + \ + '.tencentcos.cn/test.txt?acl&unsed=123' response = requests.get(url, headers={'Authorization': auth}) assert response.status_code == 200 @@ -1309,6 +1414,12 @@ def test_object_exists(): ) assert status is True + status = client.object_exists( + Bucket=test_bucket, + Key='object_not_exists' + ) + assert status is False + def test_bucket_exists(): """测试一个bucket是否存在""" @@ -1317,6 +1428,11 @@ def test_bucket_exists(): ) assert status is True + status = client.bucket_exists( + Bucket='bucket-not-exists-' + APPID + ) + assert status is False + def test_put_get_delete_bucket_policy(): """设置获取删除bucket的policy配置""" @@ -1368,7 +1484,8 @@ def test_put_file_like_object(): def test_put_chunked_object(): """支持网络流来支持chunk上传""" import requests - input = requests.get(client.get_presigned_download_url(test_bucket, test_object)) + input = requests.get( + client.get_presigned_download_url(test_bucket, test_object)) rt = client.put_object( Bucket=test_bucket, Key='test_chunked_object', @@ -1441,6 +1558,7 @@ def test_put_get_delete_bucket_domain(): Bucket=test_bucket ) + def test_put_get_delete_bucket_domain_certificate(): """测试设置获取删除bucket自定义域名证书""" @@ -1464,8 +1582,7 @@ def test_put_get_delete_bucket_domain_certificate(): ) except CosServiceError as e: if e.get_error_code() == "RecordAlreadyExist": - print(e.get_error_code()) - return + print_error_msg(e) else: raise e @@ -1496,18 +1613,26 @@ def test_put_get_delete_bucket_domain_certificate(): ) time.sleep(2) - response = client.put_bucket_domain_certificate( - Bucket=test_bucket, - DomainCertificateConfiguration=domain_cert_config - ) - # wait for sync - # get domain certificate - time.sleep(4) - response = client.get_bucket_domain_certificate( - Bucket=test_bucket, - DomainName=domain - ) - assert response["Status"] == "Enabled" + try: + response = client.put_bucket_domain_certificate( + Bucket=test_bucket, + DomainCertificateConfiguration=domain_cert_config + ) + # wait for sync + # get domain certificate + time.sleep(4) + response = client.get_bucket_domain_certificate( + Bucket=test_bucket, + DomainName=domain + ) + assert response["Status"] == "Enabled" + except CosServiceError as e: + print_error_msg(e) + response = client.get_bucket_domain_certificate( + Bucket=test_bucket, + DomainName=domain + ) + assert response["Status"] == "Disabled" # delete domain certificate response = client.delete_bucket_domain_certificate( @@ -1521,6 +1646,7 @@ def test_put_get_delete_bucket_domain_certificate(): Bucket=test_bucket, ) + def test_put_get_delete_bucket_inventory(): """测试设置获取删除bucket清单""" inventory_config = { @@ -1572,6 +1698,7 @@ def test_put_get_delete_bucket_inventory(): Id='test' ) + def test_list_bucket_inventory_configrations(): """测试列举bucket清单""" inventory_config = { @@ -1614,7 +1741,7 @@ def test_list_bucket_inventory_configrations(): Id=id, InventoryConfiguration=inventory_config, ) - + # 列举清单 i = 0 continuation_token = '' @@ -1632,7 +1759,7 @@ def test_list_bucket_inventory_configrations(): continuation_token = resp['NextContinuationToken'] else: break - + assert i == n # 删除清单 @@ -1643,6 +1770,7 @@ def test_list_bucket_inventory_configrations(): Id=id, ) + def test_put_get_delete_bucket_tagging(): """测试设置获取删除bucket标签""" tagging_config = { @@ -1873,7 +2001,7 @@ def test_select_object(): event_stream.get_select_result_to_file(file_name) if os.path.exists(file_name): os.remove(file_name) - + def test_select_event_stream_error_message(): ''' @@ -1893,24 +2021,25 @@ def test_select_event_stream_error_message(): s.write(struct.pack('>I', prelude_crc)) # Header: error-code - s.write(struct.pack('>B', 11)) # len(':error-code') + s.write(struct.pack('>B', 11)) # len(':error-code') s.write(struct.pack('11s', b':error-code')) s.write(struct.pack('>B', 7)) - s.write(struct.pack('>H', 13)) # len('InternalError') + s.write(struct.pack('>H', 13)) # len('InternalError') s.write(struct.pack('13s', b'InternalError')) # Header: error-message - s.write(struct.pack('>B', 14)) # len(':error-message') + s.write(struct.pack('>B', 14)) # len(':error-message') s.write(struct.pack('14s', b':error-message')) s.write(struct.pack('>B', 7)) - s.write(struct.pack('>H', 49)) # len('We encounted an internal error. Please try again.') + # len('We encounted an internal error. Please try again.') + s.write(struct.pack('>H', 49)) s.write(struct.pack('49s', b'We encounted an internal error. Please try again.')) # Header: message-type - s.write(struct.pack('>B', 13)) # len(':message-type') + s.write(struct.pack('>B', 13)) # len(':message-type') s.write(struct.pack('13s', b':message-type')) s.write(struct.pack('>B', 7)) - s.write(struct.pack('>H', 5)) # len('error') + s.write(struct.pack('>H', 5)) # len('error') s.write(struct.pack('5s', b'error')) # Payload @@ -1934,7 +2063,7 @@ def __init__(self): 'x-cos-request-id': 'xxx', 'x-cos-trace-id': 'yyy', } - + rt = rt_obj() event_stream = EventStream(rt) try: @@ -1958,7 +2087,8 @@ def test_get_object_sensitive_content_recognition(): Interval=3, MaxFrames=20, # BizType='xxxx', - DetectType=(CiDetectType.PORN | CiDetectType.TERRORIST | CiDetectType.POLITICS | CiDetectType.ADS), + DetectType=(CiDetectType.PORN | CiDetectType.TERRORIST | + CiDetectType.POLITICS | CiDetectType.ADS), **kwargs ) print(response) @@ -1968,7 +2098,8 @@ def test_get_object_sensitive_content_recognition(): def test_download_file(): """测试断点续传下载接口""" # 测试普通下载 - client.download_file(copy_test_bucket, test_object, 'test_download_file.local') + client.download_file(copy_test_bucket, test_object, + 'test_download_file.local') if os.path.exists('test_download_file.local'): os.remove('test_download_file.local') @@ -1976,12 +2107,14 @@ def test_download_file(): client.generate_built_in_connection_pool(10, 5) # 测试限速下载 - client.download_file(copy_test_bucket, test_object, 'test_download_traffic_limit.local', TrafficLimit='819200') + client.download_file(copy_test_bucket, test_object, + 'test_download_traffic_limit.local', TrafficLimit='819200') if os.path.exists('test_download_traffic_limit.local'): os.remove('test_download_traffic_limit.local') # 测试crc64校验开关 - client.download_file(copy_test_bucket, test_object, 'test_download_crc.local', EnableCRC=True) + client.download_file(copy_test_bucket, test_object, + 'test_download_crc.local', EnableCRC=True) if os.path.exists('test_download_crc.local'): os.remove('test_download_crc.local') @@ -2002,7 +2135,8 @@ def test_download_file(): Key=file_name ) - client.download_file(copy_test_bucket, file_name, 'test_download_md5.local') + client.download_file(copy_test_bucket, file_name, + 'test_download_md5.local') if os.path.exists('test_download_md5.local'): with open('test_download_md5.local', 'rb') as f: dest_file_md5 = get_raw_md5(f.read()) @@ -2021,12 +2155,12 @@ def test_put_get_bucket_intelligenttiering(): """测试设置获取智能分层""" try: intelligent_tiering_conf = { - 'Status': 'Enabled', - 'Transition': { - 'Days': '30', + 'Status': 'Enabled', + 'Transition': { + 'Days': '30', 'RequestFrequent': '1' - } - } + } + } response = client.put_bucket_intelligenttiering( Bucket=test_bucket, IntelligentTieringConfiguration=intelligent_tiering_conf @@ -2084,7 +2218,8 @@ def test_aes_client(): os.remove('test_for_aes_local') # 测试读取部分数据的md5 - response = client_for_aes.get_object(test_bucket, 'test_for_aes', Range='bytes=5-3000') + response = client_for_aes.get_object( + test_bucket, 'test_for_aes', Range='bytes=5-3000') response['Body'].get_stream_to_file('test_for_aes_local') with open('test_for_aes_local', 'rb') as f: local_file_md5 = get_raw_md5(f.read()) @@ -2098,12 +2233,17 @@ def test_aes_client(): content = '1' * 1024 * 1024 # 测试分片上传 client_for_rsa.delete_object(test_bucket, 'test_multi_upload') - response = client_for_aes.create_multipart_upload(test_bucket, 'test_multi_upload') + response = client_for_aes.create_multipart_upload( + test_bucket, 'test_multi_upload') uploadid = response['UploadId'] - client_for_aes.upload_part(test_bucket, 'test_multi_upload', content, 1, uploadid) - client_for_aes.upload_part(test_bucket, 'test_multi_upload', content, 2, uploadid) - response = client_for_aes.list_parts(test_bucket, 'test_multi_upload', uploadid) - client_for_aes.complete_multipart_upload(test_bucket, 'test_multi_upload', uploadid, {'Part': response['Part']}) + client_for_aes.upload_part( + test_bucket, 'test_multi_upload', content, 1, uploadid) + client_for_aes.upload_part( + test_bucket, 'test_multi_upload', content, 2, uploadid) + response = client_for_aes.list_parts( + test_bucket, 'test_multi_upload', uploadid) + client_for_aes.complete_multipart_upload( + test_bucket, 'test_multi_upload', uploadid, {'Part': response['Part']}) response = client_for_aes.get_object(test_bucket, 'test_multi_upload') response['Body'].get_stream_to_file('test_multi_upload_local') with open('test_multi_upload_local', 'rb') as f: @@ -2115,6 +2255,7 @@ def test_aes_client(): client_for_rsa.delete_object(test_bucket, 'test_multi_upload') + def test_aes_client2(): """测试aes加密客户端的上传下载操作""" aes_dir = os.path.expanduser('~/.cos_local_aes') @@ -2157,7 +2298,8 @@ def test_rsa_client(): os.remove('test_for_rsa_local') # 测试读取部分数据的md5 - response = client_for_rsa.get_object(test_bucket, 'test_for_rsa', Range='bytes=5-3000') + response = client_for_rsa.get_object( + test_bucket, 'test_for_rsa', Range='bytes=5-3000') response['Body'].get_stream_to_file('test_for_rsa_local') with open('test_for_rsa_local', 'rb') as f: local_file_md5 = get_raw_md5(f.read()) @@ -2171,12 +2313,17 @@ def test_rsa_client(): content = '1' * 1024 * 1024 # 测试分片上传 client_for_rsa.delete_object(test_bucket, 'test_multi_upload') - response = client_for_rsa.create_multipart_upload(test_bucket, 'test_multi_upload') + response = client_for_rsa.create_multipart_upload( + test_bucket, 'test_multi_upload') uploadid = response['UploadId'] - client_for_rsa.upload_part(test_bucket, 'test_multi_upload', content, 1, uploadid) - client_for_rsa.upload_part(test_bucket, 'test_multi_upload', content, 2, uploadid) - response = client_for_rsa.list_parts(test_bucket, 'test_multi_upload', uploadid) - client_for_rsa.complete_multipart_upload(test_bucket, 'test_multi_upload', uploadid, {'Part': response['Part']}) + client_for_rsa.upload_part( + test_bucket, 'test_multi_upload', content, 1, uploadid) + client_for_rsa.upload_part( + test_bucket, 'test_multi_upload', content, 2, uploadid) + response = client_for_rsa.list_parts( + test_bucket, 'test_multi_upload', uploadid) + client_for_rsa.complete_multipart_upload( + test_bucket, 'test_multi_upload', uploadid, {'Part': response['Part']}) response = client_for_rsa.get_object(test_bucket, 'test_multi_upload') response['Body'].get_stream_to_file('test_multi_upload_local') with open('test_multi_upload_local', 'rb') as f: @@ -2188,12 +2335,14 @@ def test_rsa_client(): client_for_rsa.delete_object(test_bucket, 'test_multi_upload') + def test_rsa_client2(): """测试rsa加密客户端的上传下载操作""" rsa_dir = os.path.expanduser('~/.cos_local_rsa') public_key_path = os.path.join(rsa_dir, '.public_key.pem') private_key_path = os.path.join(rsa_dir, '.private_key.pem') - rsa_provider = RSAProvider(key_pair_info=RSAProvider.get_rsa_key_pair_path(public_key_path, private_key_path)) + rsa_provider = RSAProvider(key_pair_info=RSAProvider.get_rsa_key_pair_path( + public_key_path, private_key_path)) client_for_rsa = CosEncryptionClient(conf, rsa_provider) with open('test_rsa_file', 'w') as f: @@ -2216,6 +2365,7 @@ def test_rsa_client2(): if os.path.exists('test_rsa_file'): os.remove('test_rsa_file') + def test_live_channel(): if TEST_CI != 'true': return @@ -2296,7 +2446,8 @@ def test_live_channel(): print(response) assert (response['MaxKeys'] == '10') assert (response['IsTruncated'] == 'true') - response = client.list_live_channel(Bucket=test_bucket, MaxKeys=5, Marker=response['NextMarker']) + response = client.list_live_channel( + Bucket=test_bucket, MaxKeys=5, Marker=response['NextMarker']) print(response) assert (response['MaxKeys'] == '5') assert (response['IsTruncated'] == 'true') @@ -2348,7 +2499,8 @@ def test_live_channel(): ) print("delete live channel...") - response = client.delete_live_channel(Bucket=test_bucket, ChannelName=channel_name) + response = client.delete_live_channel( + Bucket=test_bucket, ChannelName=channel_name) assert (response) @@ -2505,10 +2657,10 @@ def test_ci_get_media_queue(): # 查询媒体队列信息 kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} response = client.ci_get_media_queue( - Bucket=ci_bucket_name, - State="Active", - **kwargs - ) + Bucket=ci_bucket_name, + State="Active", + **kwargs + ) assert (response['QueueList']) @@ -2530,8 +2682,8 @@ def test_ci_create_media_transcode_watermark_jobs(): # 创建转码任务 response = client.ci_get_media_queue( - Bucket=ci_bucket_name, - State="Active", + Bucket=ci_bucket_name, + State="Active", ) QueueId = response['QueueList'][0]['QueueId'] @@ -2593,11 +2745,11 @@ def test_ci_create_media_transcode_watermark_jobs(): '' ] response = client.ci_create_media_jobs( - Bucket=ci_bucket_name, - Jobs=body, - Lst=lst, - ContentType='application/xml' - ) + Bucket=ci_bucket_name, + Jobs=body, + Lst=lst, + ContentType='application/xml' + ) assert (response['JobsDetail']) @@ -2607,8 +2759,8 @@ def test_ci_create_media_transcode_jobs(): # 创建转码任务 response = client.ci_get_media_queue( - Bucket=ci_bucket_name, - State="Active", + Bucket=ci_bucket_name, + State="Active", ) QueueId = response['QueueList'][0]['QueueId'] body = { @@ -2626,13 +2778,14 @@ def test_ci_create_media_transcode_jobs(): 'TemplateId': 't02db40900dc1c43ad9bdbd8acec6075c5' } } - kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache", "ContentType": 'application/xml'} + kwargs = {"CacheControl": "no-cache", + "ResponseCacheControl": "no-cache", "ContentType": 'application/xml'} response = client.ci_create_media_jobs( - Bucket=ci_bucket_name, - Jobs=body, - Lst={}, - **kwargs - ) + Bucket=ci_bucket_name, + Jobs=body, + Lst={}, + **kwargs + ) assert (response['JobsDetail']) @@ -2664,7 +2817,8 @@ def test_ci_create_media_pic_jobs(): }, } } - kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache", "ContentType": 'application/xml'} + kwargs = {"CacheControl": "no-cache", + "ResponseCacheControl": "no-cache", "ContentType": 'application/xml'} response = client.ci_create_media_pic_jobs( Bucket=ci_bucket_name, Jobs=body, @@ -2702,21 +2856,23 @@ def test_ci_list_media_transcode_jobs(): # 转码任务 response = client.ci_get_media_queue( - Bucket=ci_bucket_name, - State="Active", + Bucket=ci_bucket_name, + State="Active", ) QueueId = response['QueueList'][0]['QueueId'] kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} now_time = time.time() response = client.ci_list_media_jobs( - Bucket=ci_bucket_name, - QueueId=QueueId, - Tag='Transcode', - StartCreationTime=time.strftime("%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time - 5)), - EndCreationTime=time.strftime("%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time)), - Size=2, - **kwargs - ) + Bucket=ci_bucket_name, + QueueId=QueueId, + Tag='Transcode', + StartCreationTime=time.strftime( + "%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time - 5)), + EndCreationTime=time.strftime( + "%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time)), + Size=2, + **kwargs + ) assert (response) @@ -2785,33 +2941,33 @@ def test_ci_create_doc_transcode_jobs(): return kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} response = client.ci_get_doc_queue( - Bucket=ci_bucket_name, - **kwargs) + Bucket=ci_bucket_name, + **kwargs) assert (response['QueueList'][0]['QueueId']) queueId = response['QueueList'][0]['QueueId'] kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} response = client.ci_create_doc_job( - Bucket=ci_bucket_name, - QueueId=queueId, - InputObject='normal.pptx', - OutputBucket=ci_bucket_name, - OutputRegion='ap-guangzhou', - OutputObject='/test_doc/normal/abc_${Number}.jpg', - SrcType='pptx', - TgtType='jpg', - StartPage=1, - EndPage=-1, - SheetId=0, - PaperDirection=0, - PaperSize=0, - DocPassword='123', - Comments=0, - Quality=109, - Zoom=100, - ImageDpi=96, - PicPagination=1, - PageRanges='1,3', - **kwargs + Bucket=ci_bucket_name, + QueueId=queueId, + InputObject='normal.pptx', + OutputBucket=ci_bucket_name, + OutputRegion='ap-guangzhou', + OutputObject='/test_doc/normal/abc_${Number}.jpg', + SrcType='pptx', + TgtType='jpg', + StartPage=1, + EndPage=-1, + SheetId=0, + PaperDirection=0, + PaperSize=0, + DocPassword='123', + Comments=0, + Quality=109, + Zoom=100, + ImageDpi=96, + PicPagination=1, + PageRanges='1,3', + **kwargs ) assert (response['JobsDetail']['JobId']) @@ -2819,10 +2975,10 @@ def test_ci_create_doc_transcode_jobs(): JobID = response['JobsDetail']['JobId'] kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} response = client.ci_get_doc_job( - Bucket=ci_bucket_name, - JobID=JobID, - **kwargs - ) + Bucket=ci_bucket_name, + JobID=JobID, + **kwargs + ) assert (response['JobsDetail']) @@ -2831,19 +2987,21 @@ def test_ci_list_doc_transcode_jobs(): return # 查询任务列表 response = client.ci_get_doc_queue( - Bucket=ci_bucket_name - ) + Bucket=ci_bucket_name + ) assert (response['QueueList'][0]['QueueId']) queueId = response['QueueList'][0]['QueueId'] kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} now_time = time.time() response = client.ci_list_doc_jobs( - Bucket=ci_bucket_name, - QueueId=queueId, - StartCreationTime=time.strftime("%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time - 5)), - EndCreationTime=time.strftime("%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time)), - Size=10, - **kwargs + Bucket=ci_bucket_name, + QueueId=queueId, + StartCreationTime=time.strftime( + "%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time - 5)), + EndCreationTime=time.strftime( + "%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time)), + Size=10, + **kwargs ) assert response @@ -2853,38 +3011,39 @@ def test_ci_live_video_auditing(): return # 提交视频流审核任务 response = client.ci_auditing_live_video_submit( - Bucket=ci_bucket_name, - Url='rtmp://example.com/live/123', - Callback='http://callback.com/', - DataId='testdataid-111111', - UserInfo={ - 'TokenId': 'token', - 'Nickname': 'test', + Bucket=ci_bucket_name, + Url='rtmp://example.com/live/123', + Callback='http://callback.com/', + DataId='testdataid-111111', + UserInfo={ + 'TokenId': 'token', + 'Nickname': 'test', 'DeviceId': 'DeviceId-test', 'AppId': 'AppId-test', 'Room': 'Room-test', 'IP': 'IP-test', 'Type': 'Type-test', - }, - BizType='d0292362d07428b4f6982a31bf97c246', - CallbackType=1 - ) + }, + BizType='d0292362d07428b4f6982a31bf97c246', + CallbackType=1 + ) assert (response['JobsDetail']['JobId']) jobId = response['JobsDetail']['JobId'] time.sleep(5) kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} response = client.ci_auditing_live_video_cancle( - Bucket=ci_bucket_name, - JobID=jobId, - **kwargs - ) + Bucket=ci_bucket_name, + JobID=jobId, + **kwargs + ) assert (response['JobsDetail']) def test_sse_c_file(): """测试SSE-C的各种接口""" bucket = test_bucket - ssec_key = base64.standard_b64encode(to_bytes('01234567890123456789012345678901')) + ssec_key = base64.standard_b64encode( + to_bytes('01234567890123456789012345678901')) ssec_key_md5 = get_md5('01234567890123456789012345678901') file_name = 'sdk-sse-c' @@ -2892,13 +3051,15 @@ def test_sse_c_file(): response = client.put_object(Bucket=bucket, Key=file_name, Body="00000", SSECustomerAlgorithm='AES256', SSECustomerKey=ssec_key, SSECustomerKeyMD5=ssec_key_md5) print(response) - assert(response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') + assert( + response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') # 测试普通下载 response = client.get_object(Bucket=bucket, Key=file_name, SSECustomerAlgorithm='AES256', SSECustomerKey=ssec_key, SSECustomerKeyMD5=ssec_key_md5) print(response) - assert(response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') + assert( + response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') # 测试小文件高级下载 response = client.download_file(Bucket=bucket, Key=file_name, DestFilePath='sdk-sse-c.local', @@ -2911,7 +3072,8 @@ def test_sse_c_file(): # 测试普通拷贝 # 故意构造源和目标的密钥不同 - dest_ssec_key = base64.standard_b64encode(to_bytes('01234567890123456789012345678902')) + dest_ssec_key = base64.standard_b64encode( + to_bytes('01234567890123456789012345678902')) dest_ssec_key_md5 = get_md5('01234567890123456789012345678902') copy_source = {'Bucket': bucket, 'Key': file_name, 'Region': REGION} response = client.copy_object( @@ -2919,13 +3081,15 @@ def test_sse_c_file(): SSECustomerAlgorithm='AES256', SSECustomerKey=dest_ssec_key, SSECustomerKeyMD5=dest_ssec_key_md5, CopySourceSSECustomerAlgorithm='AES256', CopySourceSSECustomerKey=ssec_key, CopySourceSSECustomerKeyMD5=ssec_key_md5 ) - assert(response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') + assert( + response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') # 测试高级拷贝 response = client.copy(Bucket=bucket, Key='sdk-sse-c-copy', CopySource=copy_source, MAXThread=2, SSECustomerAlgorithm='AES256', SSECustomerKey=dest_ssec_key, SSECustomerKeyMD5=dest_ssec_key_md5, CopySourceSSECustomerAlgorithm='AES256', CopySourceSSECustomerKey=ssec_key, CopySourceSSECustomerKeyMD5=ssec_key_md5) - assert(response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') + assert( + response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') # 测试取回 response = client.put_object(Bucket=bucket, Key=file_name, Body="00000", @@ -2944,7 +3108,8 @@ def test_sse_c_file(): response = client.upload_file(Bucket=bucket, Key=file_name, LocalFilePath="sdk-sse-c-big.local", SSECustomerAlgorithm='AES256', SSECustomerKey=ssec_key, SSECustomerKeyMD5=ssec_key_md5) print(response) - assert(response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') + assert( + response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') os.remove('sdk-sse-c-big.local') # 测试大文件高级下载，走多段 @@ -2962,7 +3127,9 @@ def test_sse_c_file(): response = client.copy(Bucket=bucket, Key='sdk-sse-c-big-copy', CopySource=copy_source, MAXThread=2, SSECustomerAlgorithm='AES256', SSECustomerKey=dest_ssec_key, SSECustomerKeyMD5=dest_ssec_key_md5, StorageClass='STANDARD_IA', CopySourceSSECustomerAlgorithm='AES256', CopySourceSSECustomerKey=ssec_key, CopySourceSSECustomerKeyMD5=ssec_key_md5) - assert(response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') + assert( + response['x-cos-server-side-encryption-customer-algorithm'] == 'AES256') + def test_short_connection_put_get_object(): """使用短连接上传下载对象""" @@ -2987,6 +3154,7 @@ def test_short_connection_put_get_object(): ) assert response['Connection'] == 'close' + def test_config_invalid_scheme(): """初始化Scheme为非法值""" try: @@ -2998,6 +3166,7 @@ def test_config_invalid_scheme(): except Exception as e: print(e) + def test_config_invalid_aksk(): """aksk首尾包含空格""" try: @@ -3008,6 +3177,7 @@ def test_config_invalid_aksk(): except Exception as e: print(e) + def test_config_credential_inst(): """使用CredentialInstance初始化""" try: @@ -3018,6 +3188,7 @@ def test_config_credential_inst(): except Exception as e: raise e + def test_config_anoymous(): """匿名访问配置""" try: @@ -3028,6 +3199,7 @@ def test_config_anoymous(): except Exception as e: raise e + def test_config_none_aksk(): """缺少aksk""" try: @@ -3037,6 +3209,7 @@ def test_config_none_aksk(): except Exception as e: print(e) + def test_head_bucket_object_not_exist(): """HEAD不存在的桶和对象""" try: @@ -3060,6 +3233,7 @@ def test_head_bucket_object_not_exist(): else: raise e + def test_append_object(): """APPEND上传对象""" test_append_object = "test_append_object" @@ -3087,7 +3261,8 @@ def test_ci_delete_asr_template(): def test_ci_get_asr_template(): # 获取语音识别模板 - kwargs = {"ContentType": "application/xml", "ResponseCacheControl": "no-cache"} + kwargs = {"ContentType": "application/xml", + "ResponseCacheControl": "no-cache"} response = client.ci_get_asr_template( Bucket=ci_bucket_name, **kwargs @@ -3124,7 +3299,8 @@ def test_ci_create_asr_template(): **kwargs ) assert response - kwargs = {"ContentType": "application/xml", "ResponseCacheControl": "no-cache"} + kwargs = {"ContentType": "application/xml", + "ResponseCacheControl": "no-cache"} # 删除指定语音识别模板 response = client.ci_delete_asr_template( Bucket=ci_bucket_name, @@ -3135,7 +3311,8 @@ def test_ci_create_asr_template(): def test_ci_list_asr_jobs(): - kwargs = {"ContentType": "application/xml", "ResponseCacheControl": "no-cache"} + kwargs = {"ContentType": "application/xml", + "ResponseCacheControl": "no-cache"} response = client.ci_get_asr_queue( Bucket=ci_bucket_name, **kwargs @@ -3147,8 +3324,10 @@ def test_ci_list_asr_jobs(): response = client.ci_list_asr_jobs( Bucket=ci_bucket_name, QueueId=queueId, - StartCreationTime=time.strftime("%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time - 5)), - EndCreationTime=time.strftime("%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time)), + StartCreationTime=time.strftime( + "%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time - 5)), + EndCreationTime=time.strftime( + "%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time)), Size=10, **kwargs ) @@ -3238,7 +3417,8 @@ def test_ci_get_asr_queue(): def test_ci_get_asr_bucket(): # 查询语音识别开通状态 - kwargs = {"ContentType": "application/xml", "ResponseCacheControl": "no-cache"} + kwargs = {"ContentType": "application/xml", + "ResponseCacheControl": "no-cache"} response = client.ci_get_asr_bucket( Regions=REGION, BucketName=ci_bucket_name, @@ -3251,7 +3431,8 @@ def test_ci_get_asr_bucket(): def test_ci_get_doc_bucket(): # 查询文档预览开通状态 - kwargs = {"ContentType": "application/xml", "ResponseCacheControl": "no-cache"} + kwargs = {"ContentType": "application/xml", + "ResponseCacheControl": "no-cache"} response = client.ci_get_doc_bucket( Regions=REGION, # BucketName='demo', @@ -3265,7 +3446,8 @@ def test_ci_get_doc_bucket(): def test_ci_doc_preview_to_html_process(): # 文档预览同步接口（生成html） - kwargs = {"ContentType": "application/xml", "ResponseCacheControl": "no-cache"} + kwargs = {"ContentType": "application/xml", + "ResponseCacheControl": "no-cache"} response = client.ci_doc_preview_html_process( Bucket=ci_bucket_name, Key=ci_test_txt, @@ -3287,7 +3469,8 @@ def test_ci_doc_preview_to_html_process(): def test_ci_doc_preview_process(): # 文档预览同步接口 - kwargs = {"ContentType": "application/xml", "ResponseCacheControl": "no-cache"} + kwargs = {"ContentType": "application/xml", + "ResponseCacheControl": "no-cache"} response = client.ci_doc_preview_process( Bucket=ci_bucket_name, Key=ci_test_txt, @@ -3318,7 +3501,8 @@ def test_ci_put_doc_queue(): 'ResultFormat': 'JSON' } } - kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache", "ContentType": 'application/xml'} + kwargs = {"CacheControl": "no-cache", + "ResponseCacheControl": "no-cache", "ContentType": 'application/xml'} response = client.ci_update_doc_queue( Bucket=ci_bucket_name, QueueId=queueId, @@ -3335,8 +3519,10 @@ def test_ci_list_workflowexecution(): response = client.ci_list_workflowexecution( Bucket=ci_bucket_name, WorkflowId='w5307ee7a60d6489383c3921c715dd1c5', - StartCreationTime=time.strftime("%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time - 5)), - EndCreationTime=time.strftime("%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time)), + StartCreationTime=time.strftime( + "%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time - 5)), + EndCreationTime=time.strftime( + "%Y-%m-%dT%H:%m:%S%z", time.localtime(now_time)), **kwargs ) assert response @@ -3445,6 +3631,7 @@ def test_ci_image_detect_car(): ) assert response + def test_pic_process_when_download_object(): kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} rule = 'imageMogr2/format/jpg/interlace/1' @@ -3501,8 +3688,10 @@ def test_ci_auditing_video_submit(): jobId = response['JobsDetail']['JobId'] while True: time.sleep(5) - kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} - response = client.ci_auditing_video_query(Bucket=ci_bucket_name, JobID=jobId, **kwargs) + kwargs = {"CacheControl": "no-cache", + "ResponseCacheControl": "no-cache"} + response = client.ci_auditing_video_query( + Bucket=ci_bucket_name, JobID=jobId, **kwargs) print(response['JobsDetail']['State']) if response['JobsDetail']['State'] == 'Success': print(str(response)) @@ -3518,7 +3707,8 @@ def test_ci_auditing_audio_submit(): jobId = response['JobsDetail']['JobId'] while True: time.sleep(5) - response = client.ci_auditing_audio_query(Bucket=ci_bucket_name, JobID=jobId) + response = client.ci_auditing_audio_query( + Bucket=ci_bucket_name, JobID=jobId) print(response['JobsDetail']['State']) if response['JobsDetail']['State'] == 'Success': print(str(response)) @@ -3534,7 +3724,8 @@ def test_ci_auditing_text_submit(): jobId = response['JobsDetail']['JobId'] while True: time.sleep(5) - response = client.ci_auditing_text_query(Bucket=ci_bucket_name, JobID=jobId) + response = client.ci_auditing_text_query( + Bucket=ci_bucket_name, JobID=jobId) print(response['JobsDetail']['State']) if response['JobsDetail']['State'] == 'Success': print(str(response)) @@ -3551,8 +3742,10 @@ def test_ci_auditing_document_submit(): jobId = response['JobsDetail']['JobId'] while True: time.sleep(5) - kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} - response = client.ci_auditing_document_query(Bucket=ci_bucket_name, JobID=jobId, **kwargs) + kwargs = {"CacheControl": "no-cache", + "ResponseCacheControl": "no-cache"} + response = client.ci_auditing_document_query( + Bucket=ci_bucket_name, JobID=jobId, **kwargs) print(response['JobsDetail']['State']) print(str(response)) if response['JobsDetail']['State'] == 'Success': @@ -3569,7 +3762,8 @@ def test_ci_auditing_html_submit(): jobId = response['JobsDetail']['JobId'] while True: time.sleep(5) - response = client.ci_auditing_html_query(Bucket=ci_bucket_name, JobID=jobId) + response = client.ci_auditing_html_query( + Bucket=ci_bucket_name, JobID=jobId) print(response['JobsDetail']['State']) if response['JobsDetail']['State'] == 'Success': print(str(response)) @@ -3581,12 +3775,13 @@ def test_ci_auditing_image_batch(): kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} response = client.ci_auditing_image_batch(Bucket=ci_bucket_name, Input=[{ - 'Object': 'ocr.jpeg'}], + 'Object': 'ocr.jpeg'}], **kwargs) jobId = response['JobsDetail'][0]['JobId'] while True: time.sleep(5) - response = client.ci_auditing_image_query(Bucket=ci_bucket_name, JobID=jobId) + response = client.ci_auditing_image_query( + Bucket=ci_bucket_name, JobID=jobId) print(response['JobsDetail']['State']) if response['JobsDetail']['State'] == 'Success': print(str(response)) @@ -3603,8 +3798,10 @@ def test_ci_auditing_virus_submit(): jobId = response['JobsDetail']['JobId'] while True: time.sleep(5) - kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} - response = client.ci_auditing_virus_query(Bucket=ci_bucket_name, JobID=jobId, **kwargs) + kwargs = {"CacheControl": "no-cache", + "ResponseCacheControl": "no-cache"} + response = client.ci_auditing_virus_query( + Bucket=ci_bucket_name, JobID=jobId, **kwargs) print(response['JobsDetail']['State']) if response['JobsDetail']['State'] == 'Success': print(str(response)) @@ -3616,12 +3813,14 @@ def test_ci_auditing_detect_type(): detect_type = CiDetectType.get_detect_type_str(127) assert detect_type + def test_put_get_async_fetch_task(): from copy import deepcopy tmp_conf = deepcopy(conf) tmp_conf._scheme = 'http' tmp_client = CosS3Client(tmp_conf) - url = "{}://{}/{}".format(tmp_conf._scheme, tmp_conf.get_host(test_bucket), test_object) + url = "{}://{}/{}".format(tmp_conf._scheme, + tmp_conf.get_host(test_bucket), test_object) response = tmp_client.put_async_fetch_task( Bucket=test_bucket, FetchTaskConfiguration={ @@ -3644,6 +3843,7 @@ def test_get_rtmp_signed_url(): ) assert response + def test_change_object_storage_class(): response = client.change_object_storage_class( Bucket=test_bucket, @@ -3667,6 +3867,7 @@ def test_change_object_storage_class(): ) assert 'x-cos-storage-class' not in response + def test_update_object_meta(): response = client.update_object_meta( Bucket=test_bucket, @@ -3683,6 +3884,7 @@ def test_update_object_meta(): assert response['x-cos-meta-key1'] == 'value1' assert response['x-cos-meta-key2'] == 'value2' + def test_cos_comm_misc(): from qcloud_cos.cos_comm import format_dict_or_list, get_date, get_raw_md5, client_can_retry, format_path data = [ @@ -3703,7 +3905,7 @@ def test_cos_comm_misc(): assert r if os.path.exists("tmp_test"): os.remove("tmp_test") - + try: r = format_path('') except Exception as e: @@ -3713,10 +3915,11 @@ def test_cos_comm_misc(): r = format_path(0) except Exception as e: print(e) - + r = format_path('/test/path/to') assert r == 'test/path/to' + def test_cos_exception_unknow(): msg = '' e = CosServiceError('GET', msg, '400') @@ -3726,6 +3929,7 @@ def test_cos_exception_unknow(): assert e.get_trace_id() == 'Unknown' assert e.get_request_id() == 'Unknown' + def test_check_multipart_upload(): test_key = 'test_key' test_data = 'x'*1024*1024 @@ -3772,6 +3976,7 @@ def test_check_multipart_upload(): UploadId=uploadId ) + def test_switch_hostname_for_url(): url = "https://example-125000000.cos.ap-chengdu.myqcloud.com/123" res = switch_hostname_for_url(url) @@ -3835,7 +4040,7 @@ def test_should_switch_domain(): AutoSwitchDomainOnRetry=True, ) client1 = CosS3Client(conf1) - domain_switched = True # 已经切换过了, 本次不切换 + domain_switched = True # 已经切换过了, 本次不切换 headers = {} assert client1.should_switch_domain(domain_switched, headers) == False @@ -3864,6 +4069,7 @@ def test_should_switch_domain(): # 请求指定了ip, 不切换域名 assert client1.should_switch_domain(domain_switched, headers) == False + def test_network_failure(): """指定一个错误的ip""" conf1 = CosConfig( @@ -3884,6 +4090,7 @@ def test_network_failure(): except CosClientError as e: print(e) + if __name__ == "__main__": setUp() """ From bd0ef3b1b4335bac874a10b7fb9a1eb7cfc2fc35 Mon Sep 17 00:00:00 2001 From: flynnzzhang Date: Tue, 7 May 2024 18:32:26 +0800 Subject: [PATCH 08/20] =?UTF-8?q?ci=20ut=E8=A1=A5=E5=85=85=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qcloud_cos/cos_client.py | 5 +- ut/test.py | 406 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 400 insertions(+), 11 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 370fb2c..44ed948 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -9750,10 +9750,7 @@ def ci_delete_inventory_trigger_jobs(self, Bucket, JobId, **kwargs): params=params, headers=headers, ci_request=True) - - data = xml_to_dict(rt.content) - # 单个元素时将dict转为list - return data + return rt def ci_list_inventory_trigger_jobs(self, Bucket, StartCreationTime=None, EndCreationTime=None, OrderByTime='Desc', States='All', Size=10, diff --git a/ut/test.py b/ut/test.py index f7a1f1d..3cf212f 100644 --- a/ut/test.py +++ b/ut/test.py @@ -4,6 +4,8 @@ import time import hashlib import os +from urllib.parse import urlencode, quote + import requests import json import base64 @@ -2553,6 +2555,7 @@ def test_ci_put_image_style(): Request=body, ) assert response + time.sleep(1) body = { 'StyleName': 'style_name', } @@ -2561,6 +2564,7 @@ def test_ci_put_image_style(): Request=body, ) assert response + time.sleep(1) body = { 'StyleName': 'style_name', } @@ -3693,7 +3697,7 @@ def test_ci_auditing_video_submit(): response = client.ci_auditing_video_query( Bucket=ci_bucket_name, JobID=jobId, **kwargs) print(response['JobsDetail']['State']) - if response['JobsDetail']['State'] == 'Success': + if response['JobsDetail']['State'] == 'Success' or response['JobsDetail']['State'] == 'Failed': print(str(response)) break assert response @@ -3710,7 +3714,7 @@ def test_ci_auditing_audio_submit(): response = client.ci_auditing_audio_query( Bucket=ci_bucket_name, JobID=jobId) print(response['JobsDetail']['State']) - if response['JobsDetail']['State'] == 'Success': + if response['JobsDetail']['State'] == 'Success' or response['JobsDetail']['State'] == 'Failed': print(str(response)) break assert response @@ -3727,7 +3731,7 @@ def test_ci_auditing_text_submit(): response = client.ci_auditing_text_query( Bucket=ci_bucket_name, JobID=jobId) print(response['JobsDetail']['State']) - if response['JobsDetail']['State'] == 'Success': + if response['JobsDetail']['State'] == 'Success' or response['JobsDetail']['State'] == 'Failed': print(str(response)) break assert response @@ -3748,7 +3752,7 @@ def test_ci_auditing_document_submit(): Bucket=ci_bucket_name, JobID=jobId, **kwargs) print(response['JobsDetail']['State']) print(str(response)) - if response['JobsDetail']['State'] == 'Success': + if response['JobsDetail']['State'] == 'Success' or response['JobsDetail']['State'] == 'Failed': print(str(response)) break assert response @@ -3765,7 +3769,7 @@ def test_ci_auditing_html_submit(): response = client.ci_auditing_html_query( Bucket=ci_bucket_name, JobID=jobId) print(response['JobsDetail']['State']) - if response['JobsDetail']['State'] == 'Success': + if response['JobsDetail']['State'] == 'Success' or response['JobsDetail']['State'] == 'Failed': print(str(response)) break assert response @@ -3783,7 +3787,7 @@ def test_ci_auditing_image_batch(): response = client.ci_auditing_image_query( Bucket=ci_bucket_name, JobID=jobId) print(response['JobsDetail']['State']) - if response['JobsDetail']['State'] == 'Success': + if response['JobsDetail']['State'] == 'Success' or response['JobsDetail']['State'] == 'Failed': print(str(response)) break assert response @@ -3803,7 +3807,7 @@ def test_ci_auditing_virus_submit(): response = client.ci_auditing_virus_query( Bucket=ci_bucket_name, JobID=jobId, **kwargs) print(response['JobsDetail']['State']) - if response['JobsDetail']['State'] == 'Success': + if response['JobsDetail']['State'] == 'Success' or response['JobsDetail']['State'] == 'Failed': print(str(response)) break assert response @@ -3814,6 +3818,393 @@ def test_ci_auditing_detect_type(): assert detect_type +def test_ci_file_hash(): + """文件哈希同步请求""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.file_hash( + Bucket=ci_bucket_name, + Key=ci_test_txt, + Type='md5', + AddToHeader=True, + **kwargs + ) + assert response['FileHashCodeResult']['MD5'] == '3355b4c1078429b94a083459e194f5ec' + + +def test_ci_create_file_hash_job(): + """创建获取文件哈希值异步任务""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + body = { + 'Type': 'MD5', + } + mq_config = { + 'MqRegion': 'bj', + 'MqMode': 'Queue', + 'MqName': 'queueName' + } + response = client.ci_create_file_hash_job( + Bucket=ci_bucket_name, # 文件所在的桶名称 + InputObject=ci_test_txt, # 需要获取哈希值的文件名 + FileHashCodeConfig=body, # 获取文件哈希值配置详情 + CallBack="http://www.callback.com", # 回调url地址,当 CallBackType 参数值为 Url 时有效 + CallBackFormat="JSON", # 回调信息格式 JSON 或 XML，默认 XML + CallBackType="Url", # 回调类型，Url 或 TDMQ，默认 Url + CallBackMqConfig=mq_config, # 任务回调TDMQ配置，当 CallBackType 为 TDMQ 时必填 + UserData="this is my user data", # 透传用户信息, 可打印的 ASCII 码, 长度不超过1024 + **kwargs + ) + job_id = response['JobsDetail']['JobId'] + while True: + time.sleep(5) + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_get_file_process_jobs( + Bucket=ci_bucket_name, # 任务所在桶名称 + JobIDs=job_id, # 文件处理异步任务ID + **kwargs + ) + print(response['JobsDetail'][0]['State']) + if response['JobsDetail'][0]['State'] == 'Success' or response['JobsDetail'][0]['State'] == 'Failed': + print(str(response)) + break + assert response['JobsDetail'][0]['Operation']['FileHashCodeResult']['MD5'] == '3355b4c1078429b94a083459e194f5ec' + + +def test_ci_create_file_uncompress_job(): + """创建获取文件解压异步任务""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + body = { + 'Prefix': 'zip/result/', + 'PrefixReplaced': '0' + } + mq_config = { + 'MqRegion': 'bj', + 'MqMode': 'Queue', + 'MqName': 'queueName' + } + response = client.ci_create_file_uncompress_job( + Bucket=ci_bucket_name, # 文件所在的桶名称 + InputObject='zip/test.zip', # 需要解压的文件名 + OutputBucket=ci_bucket_name, # 指定输出文件所在的桶名称 + OutputRegion=ci_region, # 指定输出文件所在的地域 + FileUncompressConfig=body, # 文件解压配置详情 + CallBack="http://www.callback.com", # 回调url地址,当 CallBackType 参数值为 Url 时有效 + CallBackFormat="JSON", # 回调信息格式 JSON 或 XML，默认 XML + CallBackType="Url", # 回调类型，Url 或 TDMQ，默认 Url + CallBackMqConfig=mq_config, # 任务回调TDMQ配置，当 CallBackType 为 TDMQ 时必填 + UserData="this is my user data", # 透传用户信息, 可打印的 ASCII 码, 长度不超过1024 + **kwargs + ) + job_id = response['JobsDetail']['JobId'] + while True: + time.sleep(5) + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_get_file_process_jobs( + Bucket=ci_bucket_name, # 任务所在桶名称 + JobIDs=job_id, # 文件处理异步任务ID + **kwargs + ) + print(response['JobsDetail'][0]['State']) + if response['JobsDetail'][0]['State'] == 'Success' or response['JobsDetail'][0]['State'] == 'Failed': + print(str(response)) + break + assert response['JobsDetail'][0]['State'] == 'Success' + + +def test_ci_create_file_compress_job(): + """创建获取文件压缩异步任务""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + body = { + 'Flatten': '0', + 'Format': 'zip', + 'Type': 'faster', + 'Key': ['zip/result/test.txt'] + } + mq_config = { + 'MqRegion': 'bj', + 'MqMode': 'Queue', + 'MqName': 'queueName' + } + response = client.ci_create_file_compress_job( + Bucket=ci_bucket_name, # 文件所在的桶名称 + OutputBucket=ci_bucket_name, # 指定输出文件所在的桶名称 + OutputRegion=ci_region, # 指定输出文件所在的地域 + OutputObject='zip/test.zip', # 指定输出文件名 + FileCompressConfig=body, # 指定压缩配置 + CallBack="http://www.callback.com", # 回调url地址,当 CallBackType 参数值为 Url 时有效 + CallBackFormat="JSON", # 回调信息格式 JSON 或 XML，默认 XML + CallBackType="Url", # 回调类型，Url 或 TDMQ，默认 Url + CallBackMqConfig=mq_config, # 任务回调TDMQ配置，当 CallBackType 为 TDMQ 时必填 + UserData="this is my user data", # 透传用户信息, 可打印的 ASCII 码, 长度不超过1024 + **kwargs + ) + job_id = response['JobsDetail']['JobId'] + while True: + time.sleep(5) + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_get_file_process_jobs( + Bucket=ci_bucket_name, # 任务所在桶名称 + JobIDs=job_id, # 文件处理异步任务ID + **kwargs + ) + print(response['JobsDetail'][0]['State']) + if response['JobsDetail'][0]['State'] == 'Success' or response['JobsDetail'][0]['State'] == 'Failed': + print(str(response)) + break + assert response['JobsDetail'][0]['State'] == 'Success' + + +def test_ci_get_zip_preview(): + """压缩包预览同步请求""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + # 压缩包预览同步请求 + response = client.ci_file_zip_preview( + Bucket=ci_bucket_name, # 压缩文件所在桶名称 + Key="zip/test.zip", # 需要预览的压缩文件名 + **kwargs + ) + assert response['FileNumber'] == '1' + + +def test_ci_recognize_logo_process(): + """logo 识别""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_recognize_logo_process(ci_bucket_name, Key='logo.png', **kwargs) + assert response['Status'] == '0' + + +def test_ci_super_resolution_process(): + """图片超分下载时处理""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_super_resolution_process(ci_bucket_name, + Key=ci_test_car_image, + **kwargs + # Url=url + ) + assert response['Content-Type'] == 'image/jpeg' + + +def test_ci_cancel_jobs(): + """取消ci任务""" + if TEST_CI != 'true': + return + try: + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_cancel_jobs( + Bucket=ci_bucket_name, + JobID='a65xxxxxxx1f213dcd0151', + ContentType='application/xml', + **kwargs + ) + except Exception as e: + print(e) + + +def test_ci_create_inventory_trigger_jobs(): + """创建异常图片检测批量处理任务""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + body = { + 'Name': 'image-inspect-auto-move-batch-process', + 'Type': 'Job', + 'Input': { + 'Object': 'test.png', + }, + 'Operation': { + 'Tag': 'ImageInspect', + 'JobParam': { + 'ImageInspect': { + 'AutoProcess': 'false', + }, + }, + }, + } + response = client.ci_create_inventory_trigger_jobs( + Bucket=ci_bucket_name, + JobBody=body, + ContentType='application/xml', + **kwargs + ) + print(response) + assert response['JobsDetail'][0]['JobId'] is not None + job_id = response['JobsDetail'][0]['JobId'] + while True: + time.sleep(5) + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_get_inventory_trigger_jobs( + Bucket=ci_bucket_name, + JobID=job_id, + **kwargs + ) + print(response['JobsDetail'][0]['State']) + if response['JobsDetail'][0]['State'] == 'Success' or response['JobsDetail'][0]['State'] == 'Failed': + print(str(response)) + break + assert response['JobsDetail'][0]['State'] == 'Success' + + +def test_ci_delete_inventory_trigger_jobs(): + """删除批量处理任务""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + body = { + 'Name': 'image-inspect-auto-move-batch-process', + 'Type': 'Job', + 'Input': { + 'Object': 'test.png', + }, + 'Operation': { + 'Tag': 'ImageInspect', + 'JobParam': { + 'ImageInspect': { + 'AutoProcess': 'false', + }, + }, + }, + } + response = client.ci_create_inventory_trigger_jobs( + Bucket=ci_bucket_name, + JobBody=body, + ContentType='application/xml', + **kwargs + ) + print(response) + assert response['JobsDetail'][0]['JobId'] is not None + job_id = response['JobsDetail'][0]['JobId'] + response = client.ci_delete_inventory_trigger_jobs( + Bucket=ci_bucket_name, + JobId=job_id, + **kwargs + ) + print(response) + assert response + + +def test_ci_snapshot_template(): + """截图模板""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + snapshot_template_config = { + 'Name': 'snapshot_template_' + str(random.randint(0, 1000)), + 'Tag': 'Snapshot', + 'Snapshot': { + 'Mode': 'Interval', + 'Width': '128', + 'Height': '128', + 'Count': '1', + 'SnapshotOutMode': 'OnlySnapshot', + }, + } + response = client.ci_create_template( + Bucket=ci_bucket_name, + Template=snapshot_template_config, + **kwargs + ) + print(response) + assert response['Template']['TemplateId'] is not None + template_id = response['Template']['TemplateId'] + response = client.ci_update_template( + Bucket=ci_bucket_name, + TemplateId=template_id, + Template=snapshot_template_config, + **kwargs + ) + print(response) + assert response['Template']['TemplateId'] == template_id + response = client.ci_get_template( + Bucket=ci_bucket_name, + Ids=template_id, + **kwargs + ) + print(response) + assert response['TemplateList'][0]['TemplateId'] == template_id + response = client.ci_delete_template( + Bucket=ci_bucket_name, # 任务所在桶名称 + TemplateId=template_id, # 文件处理异步任务ID + **kwargs + ) + print(response) + assert response['TemplateId'] == template_id + + +def test_ci_list_inventory_trigger_jobs(): + """获取ci批量处理任务列表""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_list_inventory_trigger_jobs( + Bucket=ci_bucket_name, # 桶名称 + Type='Job', + **kwargs + ) + assert response + + +def test_ci_get_ai_bucket(): + """获取ai bucket信息""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_get_ai_bucket( + BucketName=ci_bucket_name, + **kwargs + ) + print(response) + assert response['AiBucketList']['BucketId'] == ci_bucket_name + + +def test_ci_update_ai_queue(): + """更新ai队列信息""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + + response = client.ci_get_ai_queue( + Bucket=ci_bucket_name, + ContentType='application/xml', + **kwargs + ) + assert response['QueueList'][0]['QueueId'] is not None + queue_id = response['QueueList'][0]['QueueId'] + + body = { + 'Name': 'ai-queue', + 'QueueID': queue_id, + 'State': 'Active', + 'NotifyConfig': { + 'Type': 'Url', + 'Url': 'http://www.demo.callback.com', + 'Event': 'TaskFinish', + 'State': 'On', + 'ResultFormat': 'JSON', + } + } + response = client.ci_update_ai_queue( + Bucket=ci_bucket_name, + QueueId=queue_id, + Request=body, + ContentType='application/xml', + **kwargs + ) + assert response['Queue'][0]['QueueId'] == queue_id + + def test_put_get_async_fetch_task(): from copy import deepcopy tmp_conf = deepcopy(conf) @@ -4217,5 +4608,6 @@ def test_network_failure(): test_ci_auditing_image_batch() test_ci_auditing_virus_submit() test_sse_c_file() + test_ci_file_hash() """ tearDown() From 04f7b51e654539effbaa3f80d007cd8767fe8e6f Mon Sep 17 00:00:00 2001 From: flynnzzhang Date: Wed, 8 May 2024 11:37:36 +0800 Subject: [PATCH 09/20] =?UTF-8?q?ci=20ut=E8=A1=A5=E5=85=85=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ut/test.py | 146 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 1 deletion(-) diff --git a/ut/test.py b/ut/test.py index 3cf212f..0ac29df 100644 --- a/ut/test.py +++ b/ut/test.py @@ -4189,7 +4189,7 @@ def test_ci_update_ai_queue(): 'State': 'Active', 'NotifyConfig': { 'Type': 'Url', - 'Url': 'http://www.demo.callback.com', + 'Url': 'http://www.callback.com', 'Event': 'TaskFinish', 'State': 'On', 'ResultFormat': 'JSON', @@ -4205,6 +4205,150 @@ def test_ci_update_ai_queue(): assert response['Queue'][0]['QueueId'] == queue_id +def test_ci_workflow(): + """创建/更新/获取/删除异常图片检测工作流""" + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + + body = { + 'MediaWorkflow': { + 'Name': 'image-inspect', + 'State': 'Paused', + 'Topology': { + 'Dependencies': { + 'Start': 'ImageInspectNode', + 'ImageInspectNode': 'End', + }, + 'Nodes': { + 'Start': { + 'Type': 'Start', + 'Input': { + 'ObjectPrefix': 'test', + 'NotifyConfig': { + 'Type': 'Url', + 'Url': 'http://www.callback.com', + 'Event': 'WorkflowFinish,TaskFinish', + 'ResultFormat': '', + }, + 'ExtFilter': { + 'State': 'On', + 'Image': 'true', + } + } + }, + 'ImageInspectNode': { + 'Type': 'ImageInspect', + 'Operation': { + 'ImageInspect': { + 'AutoProcess': 'true', + 'ProcessType': 'BackupObject' + } + } + }, + }, + }, + }, + } + response = client.ci_create_workflow( + Bucket=ci_bucket_name, # 桶名称 + Body=body, # 工作流配置信息 + ContentType='application/xml', + **kwargs + ) + print(response) + assert response['MediaWorkflow']['WorkflowId'] is not None + workflow_id = response['MediaWorkflow']['WorkflowId'] + + update_body = { + 'MediaWorkflow': { + 'Name': 'image-inspect', + 'State': 'Paused', + 'Topology': { + 'Dependencies': { + 'Start': 'ImageInspectNode', + 'ImageInspectNode': 'End', + }, + 'Nodes': { + 'Start': { + 'Type': 'Start', + 'Input': { + 'ObjectPrefix': 'test', + 'NotifyConfig': { + 'Type': 'Url', + 'Url': 'http://www.callback.com', + 'Event': 'WorkflowFinish,TaskFinish', + 'ResultFormat': '', + }, + 'ExtFilter': { + 'State': 'On', + 'Image': 'true', + } + } + }, + 'ImageInspectNode': { + 'Type': 'ImageInspect', + 'Operation': { + 'ImageInspect': { + 'AutoProcess': 'true', + 'ProcessType': 'SwitchObjectToPrivate' + } + } + }, + }, + }, + }, + } + + response = client.ci_update_workflow( + Bucket=ci_bucket_name, # 桶名称 + WorkflowId=workflow_id, # 需要更新的工作流ID + Body=update_body, # 工作流配置详情 + ContentType='application/xml', + **kwargs + ) + print(response) + print("workflowId is: " + response['MediaWorkflow']['WorkflowId']) + assert response['MediaWorkflow']['WorkflowId'] == workflow_id + assert response['MediaWorkflow']['Topology']['Nodes']['ImageInspectNode']['Operation']['ImageInspect']['ProcessType'] == 'SwitchObjectToPrivate' + + response = client.ci_update_workflow_state( + Bucket=ci_bucket_name, # 桶名称 + WorkflowId=workflow_id, # 需要更新的工作流ID + UpdateState='active', # 需要更新至的工作流状态，支持 active 开启 / paused 关闭 + ContentType='application/xml', + **kwargs + ) + assert response['MediaWorkflow']['State'] == 'Active' + + response = client.ci_get_workflow( + Bucket=ci_bucket_name, # 桶名称 + Ids=workflow_id, # 需要查询的工作流ID，支持传入多个，以","分隔 + Name='image-inspect', # 需要查询的工作流名称 + ContentType='application/xml', + **kwargs + ) + print(response) + assert response['MediaWorkflowList'][0]['WorkflowId'] == workflow_id + + response = client.ci_update_workflow_state( + Bucket=ci_bucket_name, # 桶名称 + WorkflowId=workflow_id, # 需要更新的工作流ID + UpdateState='paused', # 需要更新至的工作流状态，支持 active 开启 / paused 关闭 + ContentType='application/xml', + **kwargs + ) + assert response['MediaWorkflow']['State'] == 'Paused' + + response = client.ci_delete_workflow( + Bucket=ci_bucket_name, # 桶名称 + WorkflowId=workflow_id, # 需要删除的工作流ID + **kwargs + ) + print(response) + assert response['WorkflowId'] == workflow_id + + def test_put_get_async_fetch_task(): from copy import deepcopy tmp_conf = deepcopy(conf) From 676affb4efa47bf5c8167ebd8ebcbed198ef957c Mon Sep 17 00:00:00 2001 From: flynnzzhang Date: Wed, 8 May 2024 11:45:02 +0800 Subject: [PATCH 10/20] =?UTF-8?q?ci=20ut=E8=A1=A5=E5=85=85=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ut/test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ut/test.py b/ut/test.py index 0ac29df..13a717b 100644 --- a/ut/test.py +++ b/ut/test.py @@ -4349,6 +4349,22 @@ def test_ci_workflow(): assert response['WorkflowId'] == workflow_id +def test_ci_auditing_report_badcase(): + if TEST_CI != 'true': + return + kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} + response = client.ci_auditing_report_badcase(Bucket=ci_bucket_name, + ContentType=2, + Label='Ads', + SuggestedLabel='Normal', + Text=base64.b64encode("123456".encode("utf-8")).decode('utf-8'), + Url='https://' + ci_bucket_name + '.cos.ap-chongqing.myqcloud.com/ocr.jpeg', + JobId='si16ac0b3f0cec11ef9fab525400bf01fd', + ModerationTime='2024-05-08T11:36:00+08:00', + **kwargs) + assert response + + def test_put_get_async_fetch_task(): from copy import deepcopy tmp_conf = deepcopy(conf) From 05804b93a1c3ac9d7e9a5ab2af8b2e61697e22e8 Mon Sep 17 00:00:00 2001 From: flynnzzhang Date: Wed, 8 May 2024 15:56:50 +0800 Subject: [PATCH 11/20] =?UTF-8?q?ci=20ut=E8=A1=A5=E5=85=85=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ut/test.py | 67 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/ut/test.py b/ut/test.py index 13a717b..8fdbc56 100644 --- a/ut/test.py +++ b/ut/test.py @@ -78,6 +78,8 @@ def token(self): ci_test_ocr_image = "ocr.jpeg" ci_test_txt = "test.txt" ci_test_car_image = "car.jpeg" +ci_test_guanggao_audit_image = "audit_guanggao_test.jpg" +ci_test_zhengzhi_audit_image = "audit_zhengzhi_test.jpg" def _create_test_bucket(test_bucket, create_region=None): @@ -2085,16 +2087,33 @@ def test_get_object_sensitive_content_recognition(): kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} response = client.get_object_sensitive_content_recognition( Bucket=ci_bucket_name, - Key=ci_test_ocr_image, + Key=ci_test_guanggao_audit_image, Interval=3, MaxFrames=20, # BizType='xxxx', DetectType=(CiDetectType.PORN | CiDetectType.TERRORIST | - CiDetectType.POLITICS | CiDetectType.ADS), + CiDetectType.POLITICS | CiDetectType.ADS | CiDetectType.TEENAGER), + LargeImageDetect=0, + DataId="test", + CallBack="www.callback.com", **kwargs ) print(response) - assert response + assert response['AdsInfo']['Score'] != 0 + + response = client.get_object_sensitive_content_recognition( + Bucket=ci_bucket_name, + Key=ci_test_zhengzhi_audit_image, + Interval=3, + MaxFrames=20, + # BizType='xxxx', + LargeImageDetect=0, + DataId="test", + CallBack="www.callback.com", + **kwargs + ) + print(response) + assert response['PoliticsInfo']['Score'] != 0 def test_download_file(): @@ -3738,24 +3757,27 @@ def test_ci_auditing_text_submit(): def test_ci_auditing_document_submit(): - response = client.ci_auditing_document_submit(Bucket=ci_bucket_name, - Url='https://cos-python-v5-test-ci-1253960454.cos.ap-guangzhou.myqcloud.com/test.txt', - Key=ci_test_txt, - Type='txt', - Callback="http://www.demo.com") - jobId = response['JobsDetail']['JobId'] - while True: - time.sleep(5) - kwargs = {"CacheControl": "no-cache", - "ResponseCacheControl": "no-cache"} - response = client.ci_auditing_document_query( - Bucket=ci_bucket_name, JobID=jobId, **kwargs) - print(response['JobsDetail']['State']) - print(str(response)) - if response['JobsDetail']['State'] == 'Success' or response['JobsDetail']['State'] == 'Failed': + file_list = ["ads_test.docx", "politics_test.docx", "porn_test.docx", "terrorism_test.docx"] + for file in file_list: + response = client.ci_auditing_document_submit(Bucket=ci_bucket_name, + Key=file, + Type='docx', + Callback="http://www.demo.com", + DataId="test", + CallbackType=1) + jobId = response['JobsDetail']['JobId'] + while True: + time.sleep(3) + kwargs = {"CacheControl": "no-cache", + "ResponseCacheControl": "no-cache"} + response = client.ci_auditing_document_query( + Bucket=ci_bucket_name, JobID=jobId, **kwargs) + print(response['JobsDetail']['State']) print(str(response)) - break - assert response + if response['JobsDetail']['State'] == 'Success' or response['JobsDetail']['State'] == 'Failed': + print(str(response)) + break + assert response def test_ci_auditing_html_submit(): @@ -3778,8 +3800,9 @@ def test_ci_auditing_html_submit(): def test_ci_auditing_image_batch(): kwargs = {"CacheControl": "no-cache", "ResponseCacheControl": "no-cache"} response = client.ci_auditing_image_batch(Bucket=ci_bucket_name, - Input=[{ - 'Object': 'ocr.jpeg'}], + Input=[{'Object': ci_test_zhengzhi_audit_image}, + {'Object': ci_test_guanggao_audit_image}], + Callback='http://www.callback.com', **kwargs) jobId = response['JobsDetail'][0]['JobId'] while True: From aeab9b00fb71b73d94cf94d595ff6dc009d9ad44 Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Wed, 8 May 2024 19:26:01 +0800 Subject: [PATCH 12/20] =?UTF-8?q?get=5Fobject=E8=B7=AF=E5=BE=84=E6=A3=80?= =?UTF-8?q?=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qcloud_cos/cos_client.py | 6 +++- qcloud_cos/cos_comm.py | 16 +++++++++ ut/test.py | 73 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 44ed948..2c024bf 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -498,7 +498,7 @@ def put_object(self, Bucket, Body, Key, EnableMD5=False, **kwargs): response = dict(**rt.headers) return response - def get_object(self, Bucket, Key, **kwargs): + def get_object(self, Bucket, Key, KeySimplifyCheck=True, **kwargs): """单文件下载接口 :param Bucket(string): 存储桶名称. @@ -532,6 +532,10 @@ def get_object(self, Bucket, Key, **kwargs): del headers['versionId'] params = format_values(params) + # 检查key: 按照posix路径语义合并后如果是'/'则抛异常(因为这导致GetObject请求变成GetBucket) + if KeySimplifyCheck: + path_simplify_check(path=Key) + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("get object, url=:{url} ,headers=:{headers}, params=:{params}".format( url=url, diff --git a/qcloud_cos/cos_comm.py b/qcloud_cos/cos_comm.py index 1fc9368..281c954 100644 --- a/qcloud_cos/cos_comm.py +++ b/qcloud_cos/cos_comm.py @@ -331,6 +331,22 @@ def format_bucket(bucket, appid): return bucket + u"-" + appid +def path_simplify_check(path): + """将path按照posix路径语义合并后,如果结果为空或'/'则抛异常""" + path = to_unicode(path) + stack = list() + tokens = path.split(u'/') + for token in tokens: + if token == u'..': + if stack: + stack.pop() + elif token and token != u'.': + stack.append(token) + path = u'/' + u'/'.join(stack) + if path == u'/': + raise CosClientError("key is invalid") + + def format_path(path): """检查path是否合法,格式化path""" if not isinstance(path, string_types): diff --git a/ut/test.py b/ut/test.py index 8fdbc56..d187a03 100644 --- a/ut/test.py +++ b/ut/test.py @@ -4665,6 +4665,79 @@ def test_network_failure(): print(e) +def test_get_object_path_simplify_check(): + try: + response = client.get_object( + Bucket=test_bucket, + Key='' + ) + except CosClientError as e: + print(e) + + try: + response = client.get_object( + Bucket=test_bucket, + Key='/' + ) + raise Exception('err') + except CosClientError as e: + print(e) + + try: + response = client.get_object( + Bucket=test_bucket, + Key='/' + ) + raise Exception('err') + except CosClientError as e: + print(e) + + try: + response = client.get_object( + Bucket=test_bucket, + Key='////' + ) + raise Exception('err') + except CosClientError as e: + print(e) + + try: + response = client.get_object( + Bucket=test_bucket, + Key='/abc/../' + ) + raise Exception('err') + except CosClientError as e: + print(e) + + try: + response = client.get_object( + Bucket=test_bucket, + Key='/./' + ) + raise Exception('err') + except CosClientError as e: + print(e) + + try: + response = client.get_object( + Bucket=test_bucket, + Key='///abc/.//def//../../' + ) + raise Exception('err') + except CosClientError as e: + print(e) + + try: + response = client.get_object( + Bucket=test_bucket, + Key='/././///abc/.//def//../../' + ) + raise Exception('err') + except CosClientError as e: + print(e) + + if __name__ == "__main__": setUp() """ From 334f452f452501b950f9e88a38d0dc6e4aca2782 Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Wed, 8 May 2024 20:25:48 +0800 Subject: [PATCH 13/20] ut for live_channel and vod_playlist --- ut/test.py | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) diff --git a/ut/test.py b/ut/test.py index d187a03..deb08a8 100644 --- a/ut/test.py +++ b/ut/test.py @@ -2525,6 +2525,167 @@ def test_live_channel(): assert (response) +def test_live_channel_exception(): + """测试rtmp推流功能""" + livechannel_config = { + 'Description': 'cos python sdk test', + 'Switch': 'Enabled', + 'Target': { + 'Type': 'HLS', + 'FragDuration': '3', + 'FragCount': '5', + } + } + channel_name = 'cos-python-sdk-uttest-ch1' + + try: + response = client.put_live_channel( + Bucket=test_bucket, + ChannelName=channel_name, + LiveChannelConfiguration=livechannel_config) + assert (response) + except CosServiceError as e: + print(e) + + try: + response = client.list_live_channel( + Bucket=test_bucket, + Prefix='foo', + Marker='bar', + ) + assert (response) + except CosServiceError as e: + print(e) + + try: + response = client.get_live_channel_info( + Bucket=test_bucket, + ChannelName=channel_name, + ) + assert (response) + except CosServiceError as e: + print(e) + + try: + response = client.put_live_channel_switch( + Bucket=test_bucket, + ChannelName=channel_name, + Switch='enabled', + ) + assert (response) + except CosServiceError as e: + print(e) + + try: + response = client.get_live_channel_history( + Bucket=test_bucket, + ChannelName=channel_name, + ) + assert (response) + except CosServiceError as e: + print(e) + + try: + response = client.get_live_channel_status( + Bucket=test_bucket, + ChannelName=channel_name, + ) + assert (response) + except CosServiceError as e: + print(e) + + try: + response = client.delete_live_channel( + Bucket=test_bucket, + ChannelName=channel_name, + ) + assert (response) + except CosServiceError as e: + print(e) + + # vod_playlist + try: + response = client.get_vod_playlist( + Bucket=test_bucket, + ChannelName=channel_name, + StartTime=1611218201, + EndTime=1611218300, + ) + assert (response) + except CosServiceError as e: + print(e) + + try: + response = client.get_vod_playlist( + Bucket=test_bucket, + ChannelName=channel_name, + StartTime=0, + EndTime=0, + ) + assert (response) + except CosClientError as e: + print(e) + + try: + response = client.get_vod_playlist( + Bucket=test_bucket, + ChannelName=channel_name, + StartTime=1000, + EndTime=100, + ) + assert (response) + except CosClientError as e: + print(e) + + try: + response = client.post_vod_playlist( + Bucket=test_bucket, + ChannelName=channel_name, + PlaylistName='test.m3u8', + StartTime=1611218201, + EndTime=1611218300, + ) + assert (response) + except CosServiceError as e: + print(e) + + try: + response = client.post_vod_playlist( + Bucket=test_bucket, + ChannelName=channel_name, + PlaylistName='test.mp4', + StartTime=1611218201, + EndTime=1611218300, + ) + assert (response) + except CosClientError as e: + print(e) + + try: + response = client.post_vod_playlist( + Bucket=test_bucket, + ChannelName=channel_name, + PlaylistName='test.m3u8', + StartTime=0, + EndTime=0, + ) + assert (response) + except CosClientError as e: + print(e) + + try: + response = client.get_vod_playlist( + Bucket=test_bucket, + ChannelName=channel_name, + PlaylistName='test.m3u8', + StartTime=1000, + EndTime=100, + ) + assert (response) + except CosClientError as e: + print(e) + + def test_get_object_url(): """测试获取对象访问URL""" response = client.get_object_url( From 3d6308ab567a694ae83b77b0de9483cf5de5cebe Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Sat, 11 May 2024 12:28:28 +0800 Subject: [PATCH 14/20] =?UTF-8?q?download=5Ffile=E8=A1=A5=E5=85=85KeySimpl?= =?UTF-8?q?icyCheck=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qcloud_cos/cos_client.py | 4 ++-- qcloud_cos/cos_comm.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 2c024bf..6eb4940 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -3634,7 +3634,7 @@ def _check_all_upload_parts(self, bucket, key, uploadid, local_path, parts_num, already_exist_parts[part_num] = part['ETag'] return True - def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, EnableCRC=False, progress_callback=None, DumpRecordDir=None, **Kwargs): + def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, EnableCRC=False, progress_callback=None, DumpRecordDir=None, KeySimplifyCheck=True, **Kwargs): """小于等于20MB的文件简单下载，大于20MB的文件使用续传下载 :param Bucket(string): 存储桶名称. @@ -3660,7 +3660,7 @@ def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, Ena object_info = self.head_object(Bucket, Key, **head_headers) file_size = int(object_info['Content-Length']) if file_size <= 1024 * 1024 * 20: - response = self.get_object(Bucket, Key, **Kwargs) + response = self.get_object(Bucket, Key, KeySimplifyCheck, **Kwargs) response['Body'].get_stream_to_file(DestFilePath) return diff --git a/qcloud_cos/cos_comm.py b/qcloud_cos/cos_comm.py index 281c954..2d83ff3 100644 --- a/qcloud_cos/cos_comm.py +++ b/qcloud_cos/cos_comm.py @@ -344,7 +344,7 @@ def path_simplify_check(path): stack.append(token) path = u'/' + u'/'.join(stack) if path == u'/': - raise CosClientError("key is invalid") + raise CosClientError("GetObject Key is invalid") def format_path(path): From bd873221c59fb5f0a470bfe652e3afbe0858cfc2 Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Sat, 11 May 2024 15:43:09 +0800 Subject: [PATCH 15/20] =?UTF-8?q?download=5Ffile=E9=80=82=E9=85=8DKeySimpl?= =?UTF-8?q?ifyCheck?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qcloud_cos/cos_client.py | 2 +- qcloud_cos/resumable_downloader.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 6eb4940..be70ffc 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -3670,7 +3670,7 @@ def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, Ena callback = ProgressCallback(file_size, progress_callback) downloader = ResumableDownLoader(self, Bucket, Key, DestFilePath, object_info, PartSize, MAXThread, EnableCRC, - callback, DumpRecordDir, **Kwargs) + callback, DumpRecordDir, KeySimplifyCheck, **Kwargs) downloader.start() def upload_file(self, Bucket, Key, LocalFilePath, PartSize=1, MAXThread=5, EnableMD5=False, progress_callback=None, diff --git a/qcloud_cos/resumable_downloader.py b/qcloud_cos/resumable_downloader.py index a12011a..ddeabd4 100644 --- a/qcloud_cos/resumable_downloader.py +++ b/qcloud_cos/resumable_downloader.py @@ -17,7 +17,7 @@ class ResumableDownLoader(object): def __init__(self, cos_client, bucket, key, dest_filename, object_info, part_size=20, max_thread=5, - enable_crc=False, progress_callback=None, dump_record_dir = None, **kwargs): + enable_crc=False, progress_callback=None, dump_record_dir=None, key_simplify_check=True, **kwargs): self.__cos_client = cos_client self.__bucket = bucket self.__key = key @@ -27,6 +27,7 @@ def __init__(self, cos_client, bucket, key, dest_filename, object_info, part_siz self.__enable_crc = enable_crc self.__progress_callback = progress_callback self.__headers = kwargs + self.__key_simplify_check = key_simplify_check self.__max_part_count = 100 # 取决于服务端是否对并发有限制 self.__min_part_size = 1024 * 1024 # 1M @@ -130,7 +131,7 @@ def __download_part(self, part, headers): if 'TrafficLimit' in headers: traffic_limit = headers['TrafficLimit'] logger.debug("part_id: {0}, part_range: {1}, traffic_limit:{2}".format(part.part_id, range, traffic_limit)) - result = self.__cos_client.get_object(Bucket=self.__bucket, Key=self.__key, **headers) + result = self.__cos_client.get_object(Bucket=self.__bucket, Key=self.__key, KeySimplifyCheck=self.__key_simplify_check, **headers) result["Body"].pget_stream_to_file(f, part.start, part.length) self.__finish_part(part) From 98ba3c9308ac9ea9121c377f621e8f6ef8b085bf Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Sat, 11 May 2024 16:04:45 +0800 Subject: [PATCH 16/20] ut --- ut/test.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ut/test.py b/ut/test.py index deb08a8..15a189c 100644 --- a/ut/test.py +++ b/ut/test.py @@ -4899,6 +4899,44 @@ def test_get_object_path_simplify_check(): print(e) +def test_download_file_simplify_check(): + file_name = 'test_21M' + file_size = 21 + gen_file(file_name, file_size) + + key = '/abc/../' + response = client.upload_file( + Bucket=test_bucket, + Key=key, + LocalFilePath=file_name, + ) + print(response) + + response = client.download_file( + Bucket=test_bucket, + Key=key, + DestFilePath=file_name, + KeySimplifyCheck=False, + PartSize=1, + TrafficLimit='10000000', + ) + print(response) + + try: + response = client.download_file( + Bucket=test_bucket, + Key=key, + DestFilePath=file_name, + PartSize=1, + TrafficLimit='10000000', + ) + except CosClientError as e: + print(e) # 'some download_part fail after max_retry, please downloade_file again' + + if os.path.exists(file_name): + os.remove(file_name) + + if __name__ == "__main__": setUp() """ From f4df1b2105d2bdd170deda3b29baa02b4e51e2a2 Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Tue, 21 May 2024 16:23:15 +0800 Subject: [PATCH 17/20] fix ut: test_create_head_delete_maz_ofs_bucket --- ut/test.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ut/test.py b/ut/test.py index 15a189c..f085e0d 100644 --- a/ut/test.py +++ b/ut/test.py @@ -589,12 +589,19 @@ def test_create_head_delete_maz_ofs_bucket(): """创建一个多AZ OFS bucket,head它是否存在,最后删除一个空bucket""" bucket_id = str(random.randint(0, 1000)) + str(random.randint(0, 1000)) bucket_name = 'buckettest-maz-ofs' + bucket_id + '-' + APPID - response = client.create_bucket( - Bucket=bucket_name, - BucketAZConfig='MAZ', - BucketArchConfig='OFS', - ACL='public-read' - ) + try: + response = client.create_bucket( + Bucket=bucket_name, + BucketAZConfig='MAZ', + BucketArchConfig='OFS', + ACL='public-read' + ) + except CosServiceError as e: + if e.get_error_code() == 'TooManyBuckets': + return + else: + raise e + response = client.head_bucket( Bucket=bucket_name ) From 5655c636b3c6f6ba0c7280807a4fc1c9602c1f4b Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Tue, 21 May 2024 19:21:41 +0800 Subject: [PATCH 18/20] cos demo --- demo/client_encrypt.py | 128 +++++++++++++++++++ demo/copy_object.py | 79 ++++++++++++ demo/delete_object.py | 111 +++++++++++++++++ demo/download_object.py | 180 +++++++++++++++++++++++++++ demo/get_object_url.py | 32 +++++ demo/get_presigned_url.py | 226 ++++++++++++++++++++++++++++++++++ demo/head_object.py | 26 ++++ demo/list_objects.py | 62 ++++++++++ demo/modify_object_meta.py | 47 +++++++ demo/object_exists.py | 25 ++++ demo/restore_object.py | 31 +++++ demo/select_object.py | 46 +++++++ demo/server_bucket_encrypt.py | 39 ++++++ demo/server_object_encrypt.py | 136 ++++++++++++++++++++ demo/traffic_limit.py | 39 ++++++ demo/upload_object.py | 224 +++++++++++++++++++++++++++++++++ 16 files changed, 1431 insertions(+) create mode 100644 demo/client_encrypt.py create mode 100644 demo/copy_object.py create mode 100644 demo/delete_object.py create mode 100644 demo/download_object.py create mode 100644 demo/get_object_url.py create mode 100644 demo/get_presigned_url.py create mode 100644 demo/head_object.py create mode 100644 demo/list_objects.py create mode 100644 demo/modify_object_meta.py create mode 100644 demo/object_exists.py create mode 100644 demo/restore_object.py create mode 100644 demo/select_object.py create mode 100644 demo/server_bucket_encrypt.py create mode 100644 demo/server_object_encrypt.py create mode 100644 demo/traffic_limit.py create mode 100644 demo/upload_object.py diff --git a/demo/client_encrypt.py b/demo/client_encrypt.py new file mode 100644 index 0000000..aea1acf --- /dev/null +++ b/demo/client_encrypt.py @@ -0,0 +1,128 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +from qcloud_cos.cos_encryption_client import CosEncryptionClient +from qcloud_cos.crypto import AESProvider, RSAKeyPair +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 + +conf = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token) + +'''使用对称 AES256 加密每次生成的随机密钥示例 +''' + +# 方式一：通过密钥值初始化加密客户端 +# 注意：按照 AES 算法的要求，aes_key_value 需为 base64编码后的结果 +aes_provider = AESProvider(aes_key='aes_key_value') + +# 方式二：通过密钥路径初始化加密客户端 +aes_key_pair = AESProvider(aes_key_path='aes_key_path') + +client_for_aes = CosEncryptionClient(conf, aes_provider) + +# 上传对象，兼容非加密客户端的 put_object 的所有功能，具体使用可参考 put_object +response = client_for_aes.put_object( + Bucket='examplebucket-1250000000', + Body=b'bytes', + Key='exampleobject', + EnableMD5=False) + +# 下载对象，兼容非加密客户端的 get_object 的所有功能，具体使用可参考 get_object +response = client_for_aes.get_object( + Bucket='examplebucket-1250000000', + Key='exampleobject') + +# 分块上传，兼容非加密客户端的分块上传，除了最后一个part，每个 part 的大小必须为16字节的整数倍 +response = client_for_aes.create_multipart_upload( + Bucket='examplebucket-1250000000', + Key='exampleobject_upload') +uploadid = response['UploadId'] +client_for_aes.upload_part( + Bucket='examplebucket-1250000000', + Key='exampleobject_upload', + Body=b'bytes', + PartNumber=1, + UploadId=uploadid) +response = client_for_aes.list_parts( + Bucket='examplebucket-1250000000', + Key='exampleobject_upload', + UploadId=uploadid) +client_for_aes.complete_multipart_upload( + Bucket='examplebucket-1250000000', + Key='exampleobject_upload', + UploadId=uploadid, + MultipartUpload={'Part':response['Part']}) + +# 断点续传方式上传对象，`partsize`大小必须为16字节的整数倍 +response = client_for_aes.upload_file( + Bucket='test04-123456789', + LocalFilePath='local.txt', + Key='exampleobject', + PartSize=10, + MAXThread=10 +) + +'''使用非对称 RSA 加密每次生成的随机密钥示例 +''' + +# 方式一：通过密钥值初始化加密客户端 +rsa_key_pair = RSAProvider.get_rsa_key_pair('public_key_value', 'private_key_value') + +# 方式二：通过密钥路径初始化加密客户端 +rsa_key_pair = RSAProvider.get_rsa_key_pair_path('public_key_path', 'private_key_path') + +rsa_provider = RSAProvider(key_pair_info=rsa_key_pair) +client_for_rsa = CosEncryptionClient(conf, rsa_provider) + +# 上传对象，兼容非加密客户端的 put_object 的所有功能，具体使用可参考 put_object +response = client_for_rsa.put_object( + Bucket='examplebucket-1250000000', + Body=b'bytes', + Key='exampleobject', + EnableMD5=False) + +# 下载对象，兼容非加密客户端的 get_object 的所有功能，具体使用可参考 get_object +response = client_for_rsa.get_object( + Bucket='examplebucket-1250000000', + Key='exampleobject') + +# 分块上传，兼容非加密客户端的分块上传，除了最后一个 part，每个 part 的大小必须为16字节的整数倍 +response = client_for_rsa.create_multipart_upload( + Bucket='examplebucket-1250000000', + Key='exampleobject_upload') +uploadid = response['UploadId'] +client_for_rsa.upload_part( + Bucket='examplebucket-1250000000', + Key='exampleobject_upload', + Body=b'bytes', + PartNumber=1, + UploadId=uploadid) +response = client_for_rsa.list_parts( + Bucket='examplebucket-1250000000', + Key='exampleobject_upload', + UploadId=uploadid) +client_for_rsa.complete_multipart_upload( + Bucket='examplebucket-1250000000', + Key='exampleobject_upload', + UploadId=uploadid, + MultipartUpload={'Part':response['Part']}) + +# 断点续传方式上传对象，`partsize`大小必须为16字节的整数倍 +response = client_for_rsa.upload_file( + Bucket='test04-123456789', + LocalFilePath='local.txt', + Key='exampleobject', + PartSize=10, + MAXThread=10 +) \ No newline at end of file diff --git a/demo/copy_object.py b/demo/copy_object.py new file mode 100644 index 0000000..59c9deb --- /dev/null +++ b/demo/copy_object.py @@ -0,0 +1,79 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + + +'''高级接口-复制对象 +''' + +response = client.copy( + Bucket='examplebucket-1250000000', + Key='exampleobject', + CopySource={ + 'Bucket': 'sourcebucket-1250000000', + 'Key': 'sourceobject', + 'Region': 'ap-guangzhou' + } +) + +'''高级接口-移动对象 +''' + +bucket = 'examplebucket-1250000000' +srcKey = 'src_object_key' # 原始的对象路径 +destKey = 'dest_object_key' # 目的对象路径 + +# COS本身没有移动对象的接口，所谓移动就是先拷贝老对象到新对象，再删除老对象。 +response = client.copy( + Bucket=bucket, + Key=destKey, + CopySource={ + 'Bucket':bucket, + 'Key':srcKey, + 'Region':'ap-guangzhou' + }) +client.delete_object(Bucket=bucket, Key=srcKey) + +'''简单接口-复制对象 +''' + +response = client.copy_object( + Bucket='examplebucket-1250000000', + Key='exampleobject', + CopySource={ + 'Bucket': 'sourcebucket-1250000000', + 'Key': 'sourceobject', + 'Region': 'ap-guangzhou' + } +) + +'''简单接口-移动对象 +''' + +response = client.copy_object( + Bucket='examplebucket-1250000000', + Key='exampleobject', + CopySource={ + 'Bucket': 'sourcebucket-1250000000', + 'Key': 'sourceobject', + 'Region': 'ap-guangzhou' + } +) +client.delete_object(Bucket='sourcebucket-1250000000', Key='sourceobject') diff --git a/demo/delete_object.py b/demo/delete_object.py new file mode 100644 index 0000000..b1bb991 --- /dev/null +++ b/demo/delete_object.py @@ -0,0 +1,111 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +from qcloud_cos.cos_threadpool import SimpleThreadPool +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 删除单个对象 +response = client.delete_object( + Bucket='examplebucket-1250000000', + Key='exampleobject' +) +print(response) + +# 删除目录 +to_delete_dir='path/to/delete/dir/' +response = client.delete_object( + Bucket='examplebucket-1250000000', + Key=to_delete_dir +) +print(response) + +# 前缀批量删除 +# 删除指定前缀 (prefix)的文件 +bucket = 'examplebucket-1250000000' +marker = '' +prefix = 'root/logs' +while True: + response = client.list_objects(Bucket=bucket, Prefix=prefix, Marker=marker) + if 'Contents' in response: + for content in response['Contents']: + print("delete object: ", content['Key']) + client.delete_object(Bucket=bucket, Key=content['Key']) + + + if response['IsTruncated'] == 'false': + break + + + marker = response['NextMarker'] + +# 删除多个对象 +response = client.delete_objects( + Bucket='examplebucket-1250000000', + Delete={ + 'Object': [ + { + 'Key': 'exampleobject1' + }, + { + 'Key': 'exampleobject2' + } + ] + } +) + +# 批量删除对象（删除目录） + +bucket = 'examplebucket-1250000000' +folder = 'folder/' # 要删除的目录，'/'结尾表示目录 + +def delete_cos_dir(): + pool = SimpleThreadPool() + marker = "" + while True: + file_infos = [] + + # 列举一页100个对象 + response = client.list_objects(Bucket=bucket, Prefix=folder, Marker=marker, MaxKeys=100) + + if "Contents" in response: + contents = response.get("Contents") + file_infos.extend(contents) + pool.add_task(delete_files, file_infos) + + # 列举完成，退出 + if response['IsTruncated'] == 'false': + break + + # 列举下一页 + marker = response["NextMarker"] + + pool.wait_completion() + return None + + +def delete_files(file_infos): + # 构造批量删除请求 + delete_list = [] + for file in file_infos: + delete_list.append({"Key": file['Key']}) + + response = client.delete_objects(Bucket=bucket, Delete={"Object": delete_list}) + print(response) + +delete_cos_dir() \ No newline at end of file diff --git a/demo/download_object.py b/demo/download_object.py new file mode 100644 index 0000000..a1a13c1 --- /dev/null +++ b/demo/download_object.py @@ -0,0 +1,180 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +from qcloud_cos.cos_exception import CosClientError, CosServiceError +from qcloud_cos.cos_threadpool import SimpleThreadPool + +import sys +import os +import json +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在CosConfig中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +'''下载对象（断点续传） +''' + +# 使用高级接口下载一次，不重试，此时没有使用断点续传的功能 +response = client.download_file( + Bucket='examplebucket-1250000000', + Key='exampleobject', + DestFilePath='local.txt' +) + +# 使用高级接口断点续传，失败重试时不会下载已成功的分块(这里重试10次) +for i in range(0, 10): + try: + response = client.download_file( + Bucket='examplebucket-1250000000', + Key='exampleobject', + DestFilePath='local.txt') + break + except CosClientError or CosServiceError as e: + print(e) + + +'''批量下载（从COS下载目录） +''' + +# 用户的 bucket 信息 +test_bucket = 'examplebucket-1250000000' +start_prefix = 'data/' +# 对象存储依赖 分隔符 '/' 来模拟目录语义， +# 使用默认的空分隔符可以列出目录下面的所有子节点，实现类似本地目录递归的效果, +# 如果 delimiter 设置为 "/"，则需要在程序里递归处理子目录 +delimiter = '' + + +# 列出当前目录子节点，返回所有子节点信息 +def listCurrentDir(prefix): + file_infos = [] + sub_dirs = [] + marker = "" + count = 1 + while True: + response = client.list_objects(test_bucket, prefix, delimiter, marker) + # 调试输出 + # json_object = json.dumps(response, indent=4) + # print(count, " =======================================") + # print(json_object) + count += 1 + + if "CommonPrefixes" in response: + common_prefixes = response.get("CommonPrefixes") + sub_dirs.extend(common_prefixes) + + if "Contents" in response: + contents = response.get("Contents") + file_infos.extend(contents) + + if "NextMarker" in response.keys(): + marker = response["NextMarker"] + else: + break + + print("=======================================================") + + # 如果 delimiter 设置为 "/"，则需要进行递归处理子目录， + # sorted(sub_dirs, key=lambda sub_dir: sub_dir["Prefix"]) + # for sub_dir in sub_dirs: + # print(sub_dir) + # sub_dir_files = listCurrentDir(sub_dir["Prefix"]) + # file_infos.extend(sub_dir_files) + + print("=======================================================") + + sorted(file_infos, key=lambda file_info: file_info["Key"]) + for file in file_infos: + print(file) + return file_infos + + +# 下载文件到本地目录，如果本地目录已经有同名文件则会被覆盖； +# 如果目录结构不存在，则会创建和对象存储一样的目录结构 +def downLoadFiles(file_infos): + localDir = "./download/" + + pool = SimpleThreadPool() + for file in file_infos: + # 文件下载 获取文件到本地 + file_cos_key = file["Key"] + localName = localDir + file_cos_key + + # 如果本地目录结构不存在，递归创建 + if not os.path.exists(os.path.dirname(localName)): + os.makedirs(os.path.dirname(localName)) + + # skip dir, no need to download it + if str(localName).endswith("/"): + continue + + # 实际下载文件 + # 使用线程池方式 + pool.add_task(client.download_file, test_bucket, file_cos_key, localName) + + # 简单下载方式 + # response = client.get_object( + # Bucket=test_bucket, + # Key=file_cos_key, + # ) + # response['Body'].get_stream_to_file(localName) + + pool.wait_completion() + return None + + +# 功能封装，下载对象存储上面的一个目录到本地磁盘 +def downLoadDirFromCos(prefix): + global file_infos + + try: + file_infos = listCurrentDir(prefix) + + except CosServiceError as e: + print(e.get_origin_msg()) + print(e.get_digest_msg()) + print(e.get_status_code()) + print(e.get_error_code()) + print(e.get_error_msg()) + print(e.get_resource_location()) + print(e.get_trace_id()) + print(e.get_request_id()) + + downLoadFiles(file_infos) + return None + +downLoadDirFromCos(start_prefix) + + +'''简单下载 +''' + +response = client.get_object( + Bucket='examplebucket-1250000000', + Key='exampleobject' +) +response['Body'].get_stream_to_file('exampleobject') + + +'''下载对象部分内容 +''' + +response = client.get_object( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Range='bytes=0-100' +) +response['Body'].get_stream_to_file('exampleobject') + diff --git a/demo/get_object_url.py b/demo/get_object_url.py new file mode 100644 index 0000000..36d97ad --- /dev/null +++ b/demo/get_object_url.py @@ -0,0 +1,32 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging +import requests + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 生成URL +url = client.get_object_url( + Bucket='examplebucket-1250000000', + Key='exampleobject' +) +print(url) + +# 使用URL +response = requests.get(url) +print(response) \ No newline at end of file diff --git a/demo/get_presigned_url.py b/demo/get_presigned_url.py new file mode 100644 index 0000000..ccd2272 --- /dev/null +++ b/demo/get_presigned_url.py @@ -0,0 +1,226 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging +import requests + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +'''生成上传预签名 URL +''' + +# 生成上传 URL，未限制请求头部和请求参数 +url = client.get_presigned_url( + Method='PUT', + Bucket='examplebucket-1250000000', + Key='exampleobject', + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成上传 URL，同时限制存储类型和上传速度 +url = client.get_presigned_url( + Method='PUT', + Bucket='examplebucket-1250000000', + Key='exampleobject', + Headers={ + 'x-cos-storage-class':'STANDARD_IA', + 'x-cos-traffic-limit':'819200' # 预签名 URL 本身是不包含请求头部的，但请求头部会算入签名，那么使用 URL 时就必须携带请求头部，并且请求头部的值必须是这里指定的值 + }, + Expired=300 # 300秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成上传 URL，只能上传指定的文件内容 +url = client.get_presigned_url( + Method='PUT', + Bucket='examplebucket-1250000000', + Key='exampleobject', + Headers={'Content-MD5':'string'}, # 约定使用此 URL 上传对象的人必须携带 MD5 请求头部，并且请求头部的值必须是这里指定的值，这样就限定了文件的内容 + Expired=300 # 300秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成上传 URL，只能用于上传 ACL +url = client.get_presigned_url( + Method='PUT', + Bucket='examplebucket-1250000000', + Key='exampleobject', + Params={'acl':''}, # 指定了请求参数，则 URL 中会携带此请求参数，并且请求参数会算入签名，不允许使用者修改请求参数的值 + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成上传 URL，请求域名不算入签名，签名后使用者需要修改请求域名时使用 +url = client.get_presigned_url( + Method='PUT', + Bucket='examplebucket-1250000000', + Key='exampleobject', + SignHost=False, # 请求域名不算入签名，允许使用者修改请求域名，有一定安全风险 + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 使用上传 URL +response = requests.put(url=url, data=b'123') +print(response) + +'''生成下载预签名 URL +''' + +# 生成下载 URL，未限制请求头部和请求参数 +url = client.get_presigned_url( + Method='GET', + Bucket='examplebucket-1250000000', + Key='exampleobject', + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成下载 URL，同时指定响应的 content-disposition 头部，让文件在浏览器另存为，而不是显示 +url = client.get_presigned_url( + Method='GET', + Bucket='examplebucket-1250000000', + Key='exampleobject', + Params={ + 'response-content-disposition':'attachment; filename=example.xlsx' # 下载时保存为指定的文件 + # 除了 response-content-disposition，还支持 response-cache-control、response-content-encoding、response-content-language、 + # response-content-type、response-expires 等请求参数，详见下载对象 API，https://cloud.tencent.com/document/product/436/7753 + }, + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成下载 URL，同时限制下载速度 +url = client.get_presigned_url( + Method='GET', + Bucket='examplebucket-1250000000', + Key='exampleobject', + Headers={'x-cos-traffic-limit':'819200'}, # 预签名URL本身是不包含请求头部的，但请求头部会算入签名，那么使用 URL 时就必须携带请求头部，并且请求头部的值必须是这里指定的值 + Expired=300 # 300秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成下载 URL，只能用于下载 ACL +url = client.get_presigned_url( + Method='GET', + Bucket='examplebucket-1250000000', + Key='exampleobject', + Params={'acl':''}, # 指定了请求参数，则 URL 中会携带此请求参数，并且请求参数会算入签名，不允许使用者修改请求参数的值 + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成下载 URL，请求域名不算入签名，签名后使用者需要修改请求域名时使用 +url = client.get_presigned_url( + Method='GET', + Bucket='examplebucket-1250000000', + Key='exampleobject', + SignHost=False, # 请求域名不算入签名，允许使用者修改请求域名，有一定安全风险 + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 使用下载URL +response = requests.get(url) +print(response) + +'''使用临时密钥生成下载预签名 URL +''' + +# 生成下载 URL +url = client.get_presigned_url( + Method='GET', + Bucket='examplebucket-1250000000', + Key='exampleobject', + Headers={'x-cos-traffic-limit':'819200'}, # 预签名 URL 本身是不包含请求头部的，但请求头部会算入签名，那么使用 URL 时就必须携带请求头部，并且请求头部的值必须是这里指定的值 + Params={ + 'x-cos-security-token': 'string' # 使用临时密钥需要填入 Token 到请求参数 + }, + Expired=120, # 120秒后过期，过期时间请根据自身场景定义 + SignHost=False # 请求域名不算入签名，签名后使用者需要修改请求域名时使用，有一定安全风险 +) +print(url) + +# 使用下载 URL +response = requests.get(url) +print(response) + +'''生成下载对象的预签名 URL +''' + +# 生成下载 URL，未限制请求头部和请求参数 +url = client.get_presigned_download_url( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成下载 URL，同时指定响应的 content-disposition 头部，让文件在浏览器另存为，而不是显示 +url = client.get_presigned_download_url( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Params={ + 'response-content-disposition':'attachment; filename=example.xlsx' # 下载时保存为指定的文件 + # 除了 response-content-disposition，还支持 response-cache-control、response-content-encoding、response-content-language、 + # response-content-type、response-expires 等请求参数，详见下载对象 API，https://cloud.tencent.com/document/product/436/7753 + }, + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成下载 URL，同时限制下载速度 +url = client.get_presigned_download_url( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Headers={'x-cos-traffic-limit':'819200'}, # 预签名 URL 本身是不包含请求头部的，但请求头部会算入签名，那么使用 URL 时就必须携带请求头部，并且请求头部的值必须是这里指定的值 + Expired=300 # 300秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成下载 URL，只能用于下载 ACL +url = client.get_presigned_download_url( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Params={'acl':''}, # 指定了请求参数，则 URL 中会携带此请求参数，并且请求参数会算入签名，不允许使用者修改请求参数的值 + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成下载 URL，请求域名不算入签名，签名后使用者需要修改请求域名时使用 +url = client.get_presigned_download_url( + Bucket='examplebucket-1250000000', + Key='exampleobject', + SignHost=False, # 请求域名不算入签名，允许使用者修改请求域名，有一定安全风险 + Expired=120 # 120秒后过期，过期时间请根据自身场景定义 +) +print(url) + +# 生成下载 URL，使用临时密钥签名 +url = client.get_presigned_download_url( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Params={ + 'x-cos-security-token': 'string' # 使用永久密钥不需要填入 token，如果使用临时密钥需要填入 + } +) +print(url) + +# 使用下载 URL +response = requests.get(url) +print(response) \ No newline at end of file diff --git a/demo/head_object.py b/demo/head_object.py new file mode 100644 index 0000000..e6a0eed --- /dev/null +++ b/demo/head_object.py @@ -0,0 +1,26 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +response = client.head_object( + Bucket='examplebucket-1250000000', + Key='exampleobject' +) +print(response) \ No newline at end of file diff --git a/demo/list_objects.py b/demo/list_objects.py new file mode 100644 index 0000000..329a943 --- /dev/null +++ b/demo/list_objects.py @@ -0,0 +1,62 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在CosConfig中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 分页列举桶内对象，每个分页10个对象 +marker = "" +while True: + response = client.list_objects( + Bucket='examplebucket-1250000000', Prefix='folder1/', Marker=marker, MaxKeys=10) + if 'Contents' in response: + for content in response['Contents']: + print(content['Key']) + + if response['IsTruncated'] == 'false': + break + + marker = response["NextMarker"] + + +# 列举 folder1 目录下的文件和子目录 +response = client.list_objects( + Bucket='examplebucket-1250000000', Prefix='a/', Delimiter='/') +# 打印文件列表 +if 'Contents' in response: + for content in response['Contents']: + print(content['Key']) +# 打印子目录 +if 'CommonPrefixes' in response: + for folder in response['CommonPrefixes']: + print(folder['Prefix']) + + +# 查询对象及其历史版本列表 +marker = "" +while True: + response = client.list_objects_versions( + Bucket='examplebucket-1250000000', Prefix='folder1/', KeyMarker=marker, MaxKeys=10) + if 'Version' in response: + for version in response['Version']: + print(version) + + if response['IsTruncated'] == 'false': + break + + marker = response["NextMarker"] \ No newline at end of file diff --git a/demo/modify_object_meta.py b/demo/modify_object_meta.py new file mode 100644 index 0000000..36a193a --- /dev/null +++ b/demo/modify_object_meta.py @@ -0,0 +1,47 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 修改对象元数据 +response = client.copy_object( + Bucket='examplebucket-1250000000', + Key='exampleobject', + CopySource={ + 'Bucket': 'sourcebucket-1250000000', + 'Key': 'exampleobject', + 'Region': 'ap-guangzhou' + }, + CopyStatus='Replaced', + ContentType='text/plain', # 修改对象 ContentType + Metadata={'x-cos-meta-key1': 'value1', 'x-cos-meta-key2': 'value2'} # 修改自定义元数据 +) + +# 修改对象存储类型 +response = client.copy_object( + Bucket='examplebucket-1250000000', + Key='exampleobject', + CopySource={ + 'Bucket': 'examplebucket-1250000000', + 'Key': 'exampleobject', + 'Region': 'ap-guangzhou' + }, + CopyStatus='Replaced', + StorageClass='STANDARD_IA' # 修改为低频存储 +) diff --git a/demo/object_exists.py b/demo/object_exists.py new file mode 100644 index 0000000..f8f5375 --- /dev/null +++ b/demo/object_exists.py @@ -0,0 +1,25 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +response = client.object_exists( + Bucket='examplebucket-1250000000', + Key='exampleobject') +print(response) \ No newline at end of file diff --git a/demo/restore_object.py b/demo/restore_object.py new file mode 100644 index 0000000..fe0fc3e --- /dev/null +++ b/demo/restore_object.py @@ -0,0 +1,31 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的region可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +response = client.restore_object( + Bucket='examplebucket-1250000000', + Key='exampleobject', + RestoreRequest={ + 'Days': 100, + 'CASJobParameters': { + 'Tier': 'Standard' + } + } +) \ No newline at end of file diff --git a/demo/select_object.py b/demo/select_object.py new file mode 100644 index 0000000..bbefc50 --- /dev/null +++ b/demo/select_object.py @@ -0,0 +1,46 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +response = client.select_object_content( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Expression='Select * from COSObject', + ExpressionType='SQL', + InputSerialization={ + 'CompressionType': 'NONE', + 'JSON': { + 'Type': 'LINES' + } + }, + OutputSerialization={ + 'CSV': { + 'RecordDelimiter': '\n' + } + } +) + +# 获取封装在响应结果中的 EventStream 实例 +event_stream = response['Payload'] + +# 一次性获取全部检索结果 +# 注意, 因为 EventStream 对检索结果的获取是流式的, 所以当您再次调用 get_select_result() 方法时将返回空集 +result = event_stream.get_select_result() +print(result) \ No newline at end of file diff --git a/demo/server_bucket_encrypt.py b/demo/server_bucket_encrypt.py new file mode 100644 index 0000000..7bec68f --- /dev/null +++ b/demo/server_bucket_encrypt.py @@ -0,0 +1,39 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置存储桶加密 +config_dict = { + 'Rule': [ + { + 'ApplySideEncryptionConfiguration': { + 'SSEAlgorithm': 'AES256' + } + } + ] +} +response = client.put_bucket_encryption(Bucket='examplebucket-1250000000', ServerSideEncryptionConfiguration=config_dict) + +# 查询存储桶加密 +response = client.get_bucket_encryption(Bucket='examplebucket-1250000000') +sse_algorithm = response['Rule'][0]['ApplyServerSideEncryptionByDefault']['SSEAlgorithm'] + +# 删除存储桶加密 +response = client.delete_bucket_encryption(Bucket='examplebucket-1250000000') \ No newline at end of file diff --git a/demo/server_object_encrypt.py b/demo/server_object_encrypt.py new file mode 100644 index 0000000..7df19e5 --- /dev/null +++ b/demo/server_object_encrypt.py @@ -0,0 +1,136 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +from qcloud_cos.cos_comm import get_md5, to_bytes + +import sys +import os +import logging +import base64 + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的region可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 服务端加密必须使用 https + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) # 获取配置对象 +client = CosS3Client(config) + +# 上传 SSE-COS 加密对象，指定加密类型为 AES256即为 SSE-COS 加密 +response = client.put_object( + Bucket='examplebucket-125000000', + Key='sdk-sse-cos', + Body='123', + ServerSideEncryption='AES256' +) +print(response['x-cos-server-side-encryption']) # 服务端返回'x-cos-server-side-encryption': 'AES256' 表示是 SSE-COS 对象 + +# 下载 SSE-COS 加密对象，不需要携带加密头域，由服务端自动解密 +response = client.get_object( + Bucket='examplebucket-125000000', + Key='sdk-sse-cos' +) +print(response['x-cos-server-side-encryption']) # 服务端返回'x-cos-server-side-encryption': 'AES256' 表示是 SSE-COS 对象 + + +# 上传 SSE-KMS 加密对象，指定加密类型为 cos/kms 即为 SSE-KMS 加密 +response = client.put_object( + Bucket='examplebucket-125000000', + Key='sdk-sse-kms', + Body='123', + ServerSideEncryption='cos/kms', # SSE-KMS 加密固定填写 cos/kms + SSEKMSKeyId='kms-key-id', # 填写 KMS 的用户主密钥 CMK，如不填写则使用 COS 默认创建的 CMK + SSEKMSContext=base64.standard_b64encode(to_bytes("{\"test\":\"test\"}")) # 加密上下文，值为 JSON 格式加密上下文键值对的 Base64 编码，可选 +) +print(response['x-cos-server-side-encryption']) # 服务端返回'x-cos-server-side-encryption': 'cos/kms' 表示是 SSE-KMS 对象 + +# 下载 SSE-KMS 加密对象，不需要携带加密头域，由服务端自动解密 +response = client.get_object( + Bucket='examplebucket-125000000', + Key='sdk-sse-kms' +) +print(response['x-cos-server-side-encryption']) # 服务端返回'x-cos-server-side-encryption': 'cos/kms' 表示是 SSE-KMS 对象 + +# 高级接口上传 SSE-KMS 加密对象，指定加密类型为 cos/kms 即为 SSE-KMS 加密 +response = client.upload_file( + Bucket='examplebucket-125000000', + Key='sdk-sse-kms', + LocalFilePath="sdk-sse-kms.local", + ServerSideEncryption='cos/kms') # SSE-KMS 加密固定填写 cos/kms + +# 高级接口下载 SSE-KMS 加密对象，不需要携带加密头域，由服务端自动解密 +response = client.download_file( + Bucket='examplebucket-125000000', + Key='sdk-sse-kms', + DestFilePath='sdk-sse-kms.local') + +# 上传、下载、拷贝、取回 SSE-C 加密对象 + +bucket = 'examplebucket-125000000' +file_name = 'sdk-sse-c' + +# 编码和哈希加密密钥 +ssec_secret = '00000000000000000000000000000001' # 原始密钥需要是32位 +ssec_key = base64.standard_b64encode(to_bytes(ssec_secret)) # 密钥必须 Base64编码 +ssec_key_md5 = get_md5(ssec_secret) + +# 上传 SSE-C 加密对象 +response = client.put_object( + Bucket=bucket, Key=file_name, Body="00000", + SSECustomerAlgorithm='AES256', SSECustomerKey=ssec_key, SSECustomerKeyMD5=ssec_key_md5) +print(response['x-cos-server-side-encryption-customer-algorithm']) # 服务端返回'x-cos-server-side-encryption-customer-algorithm': 'AES256' 表示是 SSE-C 对象 + +# 下载 SSE-C 加密对象 +response = client.get_object( + Bucket=bucket, Key=file_name, + SSECustomerAlgorithm='AES256', SSECustomerKey=ssec_key, SSECustomerKeyMD5=ssec_key_md5) +print(response['x-cos-server-side-encryption-customer-algorithm']) # 服务端返回'x-cos-server-side-encryption-customer-algorithm': 'AES256' 表示是 SSE-C 对象 + +# HEAD SSE-C 加密对象 +response = client.head_object( + Bucket=bucket, Key=file_name, + SSECustomerAlgorithm='AES256', SSECustomerKey=ssec_key, SSECustomerKeyMD5=ssec_key_md5) +print(response['x-cos-server-side-encryption-customer-algorithm']) # 服务端返回'x-cos-server-side-encryption-customer-algorithm': 'AES256' 表示是 SSE-C 对象 + +# 高级接口上传 SSE-C 加密对象 +response = client.upload_file( + Bucket=bucket, Key=file_name, LocalFilePath="sdk-sse-c.local", + SSECustomerAlgorithm='AES256', SSECustomerKey=ssec_key, SSECustomerKeyMD5=ssec_key_md5) + +# 高级接口下载 SSE-C 加密对象 +response = client.download_file( + Bucket=bucket, Key=file_name, DestFilePath='sdk-sse-c.local', + SSECustomerAlgorithm='AES256', SSECustomerKey=ssec_key, SSECustomerKeyMD5=ssec_key_md5) + +# 拷贝 SSE-C 加密对象，源和目标的加密类型和密钥可以不一样。例子中的源对象是 SSE-C 类型，目标对象也是 SSE-C 类型。 +dest_ssec_secret = '00000000000000000000000000000002' # 密钥需要是32位 +dest_ssec_key = base64.standard_b64encode(to_bytes(dest_ssec_secret)) +dest_ssec_key_md5 = get_md5(dest_ssec_secret) +copy_source = {'Bucket': bucket, 'Key': file_name, 'Region': region} +response = client.copy_object( + Bucket=bucket, Key='sdk-sse-c-copy', CopySource=copy_source, + SSECustomerAlgorithm='AES256', SSECustomerKey=dest_ssec_key, SSECustomerKeyMD5=dest_ssec_key_md5, + CopySourceSSECustomerAlgorithm='AES256', CopySourceSSECustomerKey=ssec_key, CopySourceSSECustomerKeyMD5=ssec_key_md5 +) + +# 高级接口拷贝 SSE-C 加密对象 +response = client.copy( + Bucket=bucket, Key='sdk-sse-c-copy', CopySource=copy_source, MAXThread=2, + SSECustomerAlgorithm='AES256', SSECustomerKey=dest_ssec_key, SSECustomerKeyMD5=dest_ssec_key_md5, + CopySourceSSECustomerAlgorithm='AES256', CopySourceSSECustomerKey=ssec_key, CopySourceSSECustomerKeyMD5=ssec_key_md5) + +# 取回 SSE-C 加密对象 +response = client.restore_object( + Bucket=bucket, Key=file_name, RestoreRequest={ + 'Days': 1, + 'CASJobParameters': { + 'Tier': 'Expedited' + } + }, + SSECustomerAlgorithm='AES256', SSECustomerKey=ssec_key, SSECustomerKeyMD5=ssec_key_md5) \ No newline at end of file diff --git a/demo/traffic_limit.py b/demo/traffic_limit.py new file mode 100644 index 0000000..f73d5ff --- /dev/null +++ b/demo/traffic_limit.py @@ -0,0 +1,39 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 上传时对单链接限速 +with open('test.bin', 'rb') as fp: + response = client.put_object( + Bucket='examplebucket-1250000000', # Bucket 由 BucketName-APPID 组成 + Key='exampleobject', + Body=fp, + TrafficLimit='819200' + ) + print(response['ETag']) + + +# 下载时对单链接限速 +response = client.get_object( + Bucket='examplebucket-1250000000', + Key='exampleobject', + TrafficLimit='819200' +) +response['Body'].get_stream_to_file('exampleobject') diff --git a/demo/upload_object.py b/demo/upload_object.py new file mode 100644 index 0000000..9970a74 --- /dev/null +++ b/demo/upload_object.py @@ -0,0 +1,224 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +from qcloud_cos.cos_threadpool import SimpleThreadPool +from qcloud_cos.cos_exception import CosClientError, CosServiceError + +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +'''上传对象（断点续传） +''' + +# 使用高级接口上传一次，不重试，此时没有使用断点续传的功能 +response = client.upload_file( + Bucket='examplebucket-1250000000', + Key='exampleobject', + LocalFilePath='local.txt', + EnableMD5=False, + progress_callback=None +) + +# 使用高级接口断点续传，失败重试时不会上传已成功的分块(这里重试10次) +for i in range(0, 10): + try: + response = client.upload_file( + Bucket='examplebucket-1250000000', + Key='exampleobject', + LocalFilePath='local.txt') + break + except CosClientError or CosServiceError as e: + print(e) + + +'''批量上传（本地文件夹上传） +''' + +uploadDir = '/root/logs' +bucket = 'examplebucket-125000000' +g = os.walk(uploadDir) +# 创建上传的线程池 +pool = SimpleThreadPool() +for path, dir_list, file_list in g: + for file_name in file_list: + srcKey = os.path.join(path, file_name) + cosObjectKey = srcKey.strip('/') + # 判断 COS 上文件是否存在 + exists = False + try: + response = client.head_object(Bucket=bucket, Key=cosObjectKey) + exists = True + except CosServiceError as e: + if e.get_status_code() == 404: + exists = False + else: + print("Error happened, reupload it.") + if not exists: + print("File %s not exists in cos, upload it", srcKey) + pool.add_task(client.upload_file, bucket, cosObjectKey, srcKey) + +pool.wait_completion() +result = pool.get_result() +if not result['success_all']: + print("Not all files upload successed. you should retry") + + +'''简单文件上传 +''' + +# 文件流 简单上传 +file_name = 'test.txt' +with open('test.txt', 'rb') as fp: + response = client.put_object( + Bucket='examplebucket-1250000000', # Bucket 由 BucketName-APPID 组成 + Body=fp, + Key=file_name, + StorageClass='STANDARD', + ContentType='text/html; charset=utf-8' + ) + print(response['ETag']) + +# 字节流 简单上传 +response = client.put_object( + Bucket='examplebucket-1250000000', + Body=b'abcdefg', + Key=file_name +) +print(response['ETag']) + +# 本地路径 简单上传 +response = client.put_object_from_local_file( + Bucket='examplebucket-1250000000', + LocalFilePath='local.txt', + Key=file_name +) +print(response['ETag']) + +# 设置 HTTP 头部 简单上传 +response = client.put_object( + Bucket='examplebucket-1250000000', + Body=b'test', + Key=file_name, + ContentType='text/html; charset=utf-8' +) +print(response['ETag']) + +# 设置自定义头部 简单上传 +response = client.put_object( + Bucket='examplebucket-1250000000', + Body=b'test', + Key=file_name, + Metadata={ + 'x-cos-meta-key1': 'value1', + 'x-cos-meta-key2': 'value2' + } +) +print(response['ETag']) + +# 上传时限速 +with open('test.bin', 'rb') as fp: + response = client.put_object( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Body=fp, + TrafficLimit='819200' + ) + print(response['ETag']) + +'''查询分块上传 +''' + +response = client.list_multipart_uploads( + Bucket='examplebucket-1250000000', + Prefix='dir' +) + +'''初始化分块上传 +''' + +response = client.create_multipart_upload( + Bucket='examplebucket-1250000000', + Key='exampleobject', + StorageClass='STANDARD' +) + +'''上传分块 +''' + +# 注意，上传分块的块数最多10000块 +response = client.upload_part( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Body=b'b'*1024*1024, + PartNumber=1, + UploadId='exampleUploadId' +) + +'''复制分块 +''' + +response = client.upload_part_copy( + Bucket='examplebucket-1250000000', + Key='exampleobject', + PartNumber=1, + UploadId='exampleUploadId', + CopySource={ + 'Bucket': 'sourcebucket-1250000000', + 'Key': 'exampleobject', + 'Region': 'ap-guangzhou' + } +) + +'''查询已上传分块 +''' + +response = client.list_parts( + Bucket='examplebucket-1250000000', + Key='exampleobject', + UploadId='exampleUploadId' +) + +'''完成分块上传 +''' + +response = client.complete_multipart_upload( + Bucket='examplebucket-1250000000', + Key='exampleobject', + UploadId='exampleUploadId', + MultipartUpload={ + 'Part': [ + { + 'ETag': 'string', + 'PartNumber': 1 + }, + { + 'ETag': 'string', + 'PartNumber': 2 + }, + ] + }, +) + +'''终止分块上传 +''' + +response = client.abort_multipart_upload( + Bucket='examplebucket-1250000000', + Key='exampleobject', + UploadId='exampleUploadId' +) From 8bf07af06c35b5290f36476ba4c52211b5ff0bb1 Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Tue, 21 May 2024 20:50:39 +0800 Subject: [PATCH 19/20] cos demo --- demo/bucket_acl.py | 31 ++++++++++++++ demo/bucket_cors.py | 55 ++++++++++++++++++++++++ demo/bucket_domain.py | 45 ++++++++++++++++++++ demo/bucket_inventory.py | 85 ++++++++++++++++++++++++++++++++++++++ demo/bucket_lifecycle.py | 59 ++++++++++++++++++++++++++ demo/bucket_logging.py | 36 ++++++++++++++++ demo/bucket_policy.py | 61 +++++++++++++++++++++++++++ demo/bucket_referer.py | 47 +++++++++++++++++++++ demo/bucket_replication.py | 49 ++++++++++++++++++++++ demo/bucket_tagging.py | 45 ++++++++++++++++++++ demo/bucket_versioning.py | 38 +++++++++++++++++ demo/bucket_website.py | 61 +++++++++++++++++++++++++++ demo/object_acl.py | 33 +++++++++++++++ demo/object_tagging.py | 50 ++++++++++++++++++++++ 14 files changed, 695 insertions(+) create mode 100644 demo/bucket_acl.py create mode 100644 demo/bucket_cors.py create mode 100644 demo/bucket_domain.py create mode 100644 demo/bucket_inventory.py create mode 100644 demo/bucket_lifecycle.py create mode 100644 demo/bucket_logging.py create mode 100644 demo/bucket_policy.py create mode 100644 demo/bucket_referer.py create mode 100644 demo/bucket_replication.py create mode 100644 demo/bucket_tagging.py create mode 100644 demo/bucket_versioning.py create mode 100644 demo/bucket_website.py create mode 100644 demo/object_acl.py create mode 100644 demo/object_tagging.py diff --git a/demo/bucket_acl.py b/demo/bucket_acl.py new file mode 100644 index 0000000..8519ace --- /dev/null +++ b/demo/bucket_acl.py @@ -0,0 +1,31 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置存储桶 ACL +response = client.put_bucket_acl( + Bucket='examplebucket-1250000000', + ACL='public-read' +) + +# 查询存储桶 ACL +response = client.get_bucket_acl( + Bucket='examplebucket-1250000000' +) diff --git a/demo/bucket_cors.py b/demo/bucket_cors.py new file mode 100644 index 0000000..1b5c978 --- /dev/null +++ b/demo/bucket_cors.py @@ -0,0 +1,55 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置跨域配置 +response = client.put_bucket_cors( + Bucket='examplebucket-1250000000', + CORSConfiguration={ + 'CORSRule': [ + { + 'ID': 'string', + 'MaxAgeSeconds': 100, + 'AllowedOrigin': [ + 'string', + ], + 'AllowedMethod': [ + 'string', + ], + 'AllowedHeader': [ + 'string', + ], + 'ExposeHeader': [ + 'string', + ] + } + ] + }, +) + +# 查询跨域配置 +response = client.get_bucket_cors( + Bucket='examplebucket-1250000000', +) + +# 删除跨域配置 +response = client.delete_bucket_cors( + Bucket='examplebucket-1250000000', +) diff --git a/demo/bucket_domain.py b/demo/bucket_domain.py new file mode 100644 index 0000000..990d4a6 --- /dev/null +++ b/demo/bucket_domain.py @@ -0,0 +1,45 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置自定义域名 +response = client.put_bucket_domain( + Bucket='bucket', + DomainConfiguration={ + 'DomainRule': [ + { + 'Name': 'example.com', + 'Type': 'REST'|'WEBSITE'|'ACCELERATE', + 'Status': 'ENABLED'|'DISABLED', + 'ForcedReplacement': 'CNAME'|'TXT' + }, + ] + } +) + +# 查询自定义域名 +response = client.get_bucket_domain( + Bucket='examplebucket-1250000000' +) + +# 删除自定义域名 +response = client.delete_bucket_domain( + Bucket='examplebucket-1250000000' +) diff --git a/demo/bucket_inventory.py b/demo/bucket_inventory.py new file mode 100644 index 0000000..e89a130 --- /dev/null +++ b/demo/bucket_inventory.py @@ -0,0 +1,85 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置清单任务 +response = client.put_bucket_inventory( + Bucket='examplebucket-1250000000', + Id='string', + InventoryConfiguration={ + 'Destination': { + 'COSBucketDestination': { + 'AccountId': '100000000001', + 'Bucket': 'qcs::cos:ap-guangzhou::examplebucket-1250000000', + 'Format': 'CSV', + 'Prefix': 'string', + 'Encryption': { + 'SSECOS': {} + } + } + }, + 'IsEnabled': 'true'|'false', + 'Filter': { + 'Prefix': 'string' + }, + 'IncludedObjectVersions':'All'|'Current', + 'OptionalFields': { + 'Field': [ + 'Size', + 'LastModifiedDate', + 'ETag', + 'StorageClass', + 'IsMultipartUploaded', + 'ReplicationStatus' + ] + }, + 'Schedule': { + 'Frequency': 'Daily'|'Weekly' + } + } +) + +# 查询清单任务 +response = client.get_bucket_inventory( + Bucket='examplebucket-1250000000', + Id='string' +) + +# 列举清单任务 +continuation_token = '' +while True: + resp = client.list_bucket_inventory_configurations( + Bucket='examplebucket-1250000000', + ContinuationToken=continuation_token, + ) + if 'InventoryConfiguration' in resp: + for conf in resp['InventoryConfiguration']: + print(conf) + + if resp['IsTruncated'] == 'true': + continuation_token = resp['NextContinuationToken'] + else: + break + +# 删除清单任务 +response = client.delete_bucket_inventory( + Bucket='examplebucket-1250000000', + Id='string' +) diff --git a/demo/bucket_lifecycle.py b/demo/bucket_lifecycle.py new file mode 100644 index 0000000..552baa1 --- /dev/null +++ b/demo/bucket_lifecycle.py @@ -0,0 +1,59 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置生命周期 +response = client.put_bucket_lifecycle( + Bucket='examplebucket-1250000000', + LifecycleConfiguration={ + 'Rule': [ + { + 'ID': 'string', # 设置规则的 ID，例如Rule-1 + 'Filter': { + 'Prefix': '' # 配置前缀为空，桶内所有对象都会执行此规则 + }, + 'Status': 'Enabled', # Enabled 表示启用规则 + 'Expiration': { + 'Days': 200 # 设置对象的当前版本200天后过期删除 + }, + 'Transition': [ + { + 'Days': 100, # 设置对象的当前版本100天后沉降 + 'StorageClass': 'Standard_IA' # 沉降为低频存储 + }, + ], + 'AbortIncompleteMultipartUpload': { + 'DaysAfterInitiation': 7 # 设置7天后回收未合并的分块 + } + } + ] + } +) + +# 查询生命周期 +response = client.get_bucket_lifecycle( + Bucket='examplebucket-1250000000', +) + +# 删除生命周期 +response = client.delete_bucket_lifecycle( + Bucket='examplebucket-1250000000', +) + diff --git a/demo/bucket_logging.py b/demo/bucket_logging.py new file mode 100644 index 0000000..5e9a3f2 --- /dev/null +++ b/demo/bucket_logging.py @@ -0,0 +1,36 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置日志管理 +response = client.put_bucket_logging( + Bucket='examplebucket-1250000000', + BucketLoggingStatus={ + 'LoggingEnabled': { + 'TargetBucket': 'logging-bucket-1250000000', + 'TargetPrefix': 'string' + } + } +) + +# 查询日志管理 +response = client.get_bucket_logging( + Bucket='examplebucket-1250000000' +) diff --git a/demo/bucket_policy.py b/demo/bucket_policy.py new file mode 100644 index 0000000..f34e79d --- /dev/null +++ b/demo/bucket_policy.py @@ -0,0 +1,61 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging +import json + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置存储桶策略 +response = client.put_bucket_policy( + Bucket='examplebucket-1250000000', + Policy={ + "Statement": [ + { + "Principal": { + "qcs": [ + "qcs::cam::uin/100000000001:uin/100000000011" + ] + }, + "Effect": "allow", + "Action": [ + "name/cos:GetBucket" + ], + "Resource": [ + "qcs::cos:ap-guangzhou:uid/1250000000:examplebucket-1250000000/*" + ], + "condition": { + "ip_equal": { + "qcs:ip": "10.121.2.10/24" + } + } + } + ], + "version": "2.0" + } +) + +# 查询存储桶策略 +response = client.get_bucket_policy( + Bucket='examplebucket-1250000000', +) +policy = json.loads(response['Policy']) + +# 删除存储桶策略 +response = client.delete_bucket_policy( + Bucket='examplebucket-1250000000', +) diff --git a/demo/bucket_referer.py b/demo/bucket_referer.py new file mode 100644 index 0000000..8048014 --- /dev/null +++ b/demo/bucket_referer.py @@ -0,0 +1,47 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置存储桶 Referer +referer_config = { + 'Status': 'Enabled', + 'RefererType': 'White-List', + 'EmptyReferConfiguration': 'Allow', + 'DomainList': { + 'Domain': [ + '*.qq.com', + '*.qcloud.com' + ] + } +} +response = client.put_bucket_referer( + Bucket='examplebucket-1250000000', + RefererConfiguration=referer_config +) + +# 查询存储桶 Referer +response = client.get_bucket_referer( + Bucket='examplebucket-1250000000' +) + +# 删除存储桶 Referer +response = client.delete_bucket_referer( + Bucket='examplebucket-1250000000' +) \ No newline at end of file diff --git a/demo/bucket_replication.py b/demo/bucket_replication.py new file mode 100644 index 0000000..1ab90e5 --- /dev/null +++ b/demo/bucket_replication.py @@ -0,0 +1,49 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置存储桶复制 +response = client.put_bucket_replication( + Bucket='examplebucket-1250000000', + ReplicationConfiguration={ + 'Role': 'qcs::cam::uin/100000000001:uin/100000000001', + 'Rule': [ + { + 'ID': 'string', + 'Status': 'Enabled', + 'Destination': { + 'Bucket': 'qcs::cos:ap-shanghai::destinationbucket-1250000000', + 'StorageClass': 'STANDARD' + } + } + ] + } +) + +# 查询存储桶复制 +response = client.get_bucket_replication( + Bucket='examplebucket-1250000000' +) +print(response) + +# 删除存储桶复制 +response = client.delete_bucket_replication( + Bucket='examplebucket-1250000000' +) diff --git a/demo/bucket_tagging.py b/demo/bucket_tagging.py new file mode 100644 index 0000000..33d104c --- /dev/null +++ b/demo/bucket_tagging.py @@ -0,0 +1,45 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置存储桶标签 +response = client.put_bucket_tagging( + Bucket='examplebucket-1250000000', + Tagging={ + 'TagSet': { + 'Tag': [ + { + 'Key': 'string', + 'Value': 'string' + }, + ] + } + } +) + +# 查询存储桶标签 +response = client.get_bucket_tagging( + Bucket='examplebucket-1250000000' +) + +# 删除存储桶标签 +response = client.delete_bucket_tagging( + Bucket='examplebucket-1250000000' +) diff --git a/demo/bucket_versioning.py b/demo/bucket_versioning.py new file mode 100644 index 0000000..ff6890a --- /dev/null +++ b/demo/bucket_versioning.py @@ -0,0 +1,38 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 开启版本控制 +response = client.put_bucket_versioning( + Bucket='examplebucket-1250000000', + Status='Enabled' +) + +# 暂停版本控制 +response = client.put_bucket_versioning( + Bucket='examplebucket-1250000000', + Status='Suspended' +) + +# 查询版本控制 +response = client.get_bucket_versioning( + Bucket='examplebucket-1250000000', +) +print(response) diff --git a/demo/bucket_website.py b/demo/bucket_website.py new file mode 100644 index 0000000..0521fbe --- /dev/null +++ b/demo/bucket_website.py @@ -0,0 +1,61 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置静态网站 +response = client.put_bucket_website( + Bucket='bucket', + WebsiteConfiguration={ + 'IndexDocument': { + 'Suffix': 'string' + }, + 'ErrorDocument': { + 'Key': 'string' + }, + 'RedirectAllRequestsTo': { + 'Protocol': 'http'|'https' + }, + 'RoutingRules': [ + { + 'Condition': { + 'HttpErrorCodeReturnedEquals': 'string', + 'KeyPrefixEquals': 'string' + }, + 'Redirect': { + 'HttpRedirectCode': 'string', + 'Protocol': 'http'|'https', + 'ReplaceKeyPrefixWith': 'string', + 'ReplaceKeyWith': 'string' + } + } + ] + } +) + +# 查询静态网站配置 +response = client.get_bucket_website( + Bucket='examplebucket-1250000000' +) + +# 删除静态网站配置 +response = client.delete_bucket_website( + Bucket='examplebucket-1250000000' +) + diff --git a/demo/object_acl.py b/demo/object_acl.py new file mode 100644 index 0000000..f9794eb --- /dev/null +++ b/demo/object_acl.py @@ -0,0 +1,33 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在CosConfig中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的region可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置对象 ACL +response = client.put_object_acl( + Bucket='examplebucket-1250000000', + Key='exampleobject', + ACL='public-read' +) + +# 查询对象 ACL +response = client.get_object_acl( + Bucket='examplebucket-1250000000', + Key='exampleobject' +) diff --git a/demo/object_tagging.py b/demo/object_tagging.py new file mode 100644 index 0000000..370658a --- /dev/null +++ b/demo/object_tagging.py @@ -0,0 +1,50 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用 INFO，需要定位时可以修改为 DEBUG，此时 SDK 会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region 等。Appid 已在 CosConfig 中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的 region 可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS 支持的所有 region 列表参见 https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入 token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见 https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 设置对象标签 +response = client.put_object_tagging( + Bucket='examplebucket-1250000000', + Key='exampleobject', + Tagging={ + 'TagSet': { + 'Tag': [ + { + 'Key': "string", + 'Value': 'string' + } + ] + } + } +) + +# 查询对象标签 +response = client.get_object_tagging( + Bucket='examplebucket-1250000000', + Key='exampleobject' +) +print(response) + +# 删除对象标签 +response = client.delete_object_tagging( + Bucket='examplebucket-1250000000', + Key='exampleobject' +) + From 103ab945aa97baa0148e63cd2d0bab32feb8c50a Mon Sep 17 00:00:00 2001 From: libertyzhu Date: Wed, 22 May 2024 16:29:08 +0800 Subject: [PATCH 20/20] bucket operation demo --- demo/bucket_operation.py | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 demo/bucket_operation.py diff --git a/demo/bucket_operation.py b/demo/bucket_operation.py new file mode 100644 index 0000000..c90596c --- /dev/null +++ b/demo/bucket_operation.py @@ -0,0 +1,46 @@ +# -*- coding=utf-8 +from qcloud_cos import CosConfig +from qcloud_cos import CosS3Client +import sys +import os +import logging + +# 正常情况日志级别使用INFO，需要定位时可以修改为DEBUG，此时SDK会打印和服务端的通信信息 +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + +# 1. 设置用户属性, 包括 secret_id, secret_key, region等。Appid 已在CosConfig中移除，请在参数 Bucket 中带上 Appid。Bucket 由 BucketName-Appid 组成 +secret_id = os.environ['COS_SECRET_ID'] # 用户的 SecretId，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +secret_key = os.environ['COS_SECRET_KEY'] # 用户的 SecretKey，建议使用子账号密钥，授权遵循最小权限指引，降低使用风险。子账号密钥获取可参见 https://cloud.tencent.com/document/product/598/37140 +region = 'ap-beijing' # 替换为用户的 region，已创建桶归属的region可以在控制台查看，https://console.cloud.tencent.com/cos5/bucket + # COS支持的所有region列表参见https://cloud.tencent.com/document/product/436/6224 +token = None # 如果使用永久密钥不需要填入token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见https://cloud.tencent.com/document/product/436/14048 +scheme = 'https' # 指定使用 http/https 协议来访问 COS，默认为 https，可不填 + +config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Scheme=scheme) +client = CosS3Client(config) + +# 查询存储桶列表 +response = client.list_buckets() +print(response) + +# 创建存储桶 +# 存储桶名称不支持大写字母，COS 后端会将用户传入的大写字母自动转换为小写字母用于创建存储桶 +response = client.create_bucket( + Bucket='examplebucket-1250000000' +) + +# 检索存储桶及其权限 +response = client.head_bucket( + Bucket='examplebucket-1250000000' +) + +# 删除存储桶 +response = client.delete_bucket( + Bucket='examplebucket-1250000000' +) + +# 判断存储桶是否存在 +response = client.bucket_exists( + Bucket='examplebucket-1250000000' +) +print(response)