Merge pull request #324 from tencentyun/feature_libertyzhu_7a81e6b8

Feature libertyzhu 7a81e6b8

Author: l-iberty

qcloud_cos/cos_client.py

@@ -4037,7 +4037,8 @@ def _check_all_upload_parts(self, bucket, key, uploadid, local_path, parts_num,
             already_exist_parts[part_num] = part['ETag']
         return True
 
-    def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, EnableCRC=False, progress_callback=None, DumpRecordDir=None, KeySimplifyCheck=True, DisableTempDestFilePath=False, **Kwargs):
+    def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, EnableCRC=False, progress_callback=None,
+                      DumpRecordDir=None, KeySimplifyCheck=True, DisableTempDestFilePath=False, MaxPartCount=10000, **Kwargs):
         """小于等于20MB的文件简单下载，大于20MB的文件使用续传下载
 
         :param Bucket(string): 存储桶名称.
@@ -4049,10 +4050,11 @@ def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, Ena
         :param DumpRecordDir(string): 指定保存断点信息的文件路径
         :param KeySimplifyCheck(bool): 是否对Key进行posix路径语义归并检查
         :param DisableTempDestFilePath(bool): 简单下载写入目标文件时,不使用临时文件
+        :param MaxPartCount(int): 分块下载的最大分块数
         :param kwargs(dict): 设置请求headers.
         """
         logger.debug("Start to download file, bucket: {0}, key: {1}, dest_filename: {2}, part_size: {3}MB,\
-                     max_thread: {4}".format(Bucket, Key, DestFilePath, PartSize, MAXThread))
+                     max_thread: {4}, max_part_count: {5}".format(Bucket, Key, DestFilePath, PartSize, MAXThread, MaxPartCount))
 
         head_headers = dict()
         # SSE-C对象在head时也要求传入加密头域
@@ -4075,7 +4077,7 @@ def download_file(self, Bucket, Key, DestFilePath, PartSize=20, MAXThread=5, Ena
         if progress_callback:
             callback = ProgressCallback(file_size, progress_callback)
 
-        downloader = ResumableDownLoader(self, Bucket, Key, DestFilePath, object_info, PartSize, MAXThread, EnableCRC,
+        downloader = ResumableDownLoader(self, Bucket, Key, DestFilePath, object_info, PartSize, MAXThread, MaxPartCount, EnableCRC,
                                          callback, DumpRecordDir, KeySimplifyCheck, **Kwargs)
         downloader.start()
 

qcloud_cos/cos_comm.py

@@ -37,6 +37,7 @@
     'ResponseContentEncoding': 'response-content-encoding',
     'Metadata': 'Metadata',
     'ACL': 'x-cos-acl',
+    'Tagging': 'x-cos-tagging',
     'GrantFullControl': 'x-cos-grant-full-control',
     'GrantWrite': 'x-cos-grant-write',
     'GrantRead': 'x-cos-grant-read',
@@ -63,7 +64,8 @@
     'Referer': 'Referer',
     'PicOperations': 'Pic-Operations',
     'TrafficLimit': 'x-cos-traffic-limit',
-    'Accept': 'Accept'
+    'Accept': 'Accept',
+    'AcceptEncoding': 'Accept-Encoding',
 }
 
 

qcloud_cos/cos_exception.py

@@ -1,6 +1,7 @@
 # -*- coding=utf-8
 
 import xml.dom.minidom
+from requests.structures import CaseInsensitiveDict
 
 
 class CosException(Exception):
@@ -51,7 +52,7 @@ class CosServiceError(CosException):
 
     def __init__(self, method, message, status_code):
         CosException.__init__(self, message)
-        if isinstance(message, dict):
+        if isinstance(message, dict) or isinstance(message, CaseInsensitiveDict):
             self._origin_msg = ''
             self._digest_msg = message
         else:

qcloud_cos/resumable_downloader.py

@@ -18,7 +18,7 @@
 
 class ResumableDownLoader(object):
     def __init__(self, cos_client, bucket, key, dest_filename, object_info, part_size=20, max_thread=5,
-                 enable_crc=False, progress_callback=None, dump_record_dir=None, key_simplify_check=True, **kwargs):
+                 max_part_count=100, enable_crc=False, progress_callback=None, dump_record_dir=None, key_simplify_check=True, **kwargs):
         self.__cos_client = cos_client
         self.__bucket = bucket
         self.__key = key
@@ -30,7 +30,7 @@ def __init__(self, cos_client, bucket, key, dest_filename, object_info, part_siz
         self.__headers = kwargs
         self.__key_simplify_check = key_simplify_check
 
-        self.__max_part_count = 100  # 取决于服务端是否对并发有限制
+        self.__max_part_count = max_part_count  # 取决于服务端是否对并发有限制
         self.__min_part_size = 1024 * 1024  # 1M
         self.__part_size = self.__determine_part_size_internal(int(object_info['Content-Length']), part_size)
         self.__finished_parts = []

ut/test.py

@@ -78,6 +78,9 @@ def token(self):
     SecretKey=SECRET_KEY,
 )
 
+anonymous_conf = CosConfig(Appid=APPID, Region=REGION, Anonymous=True)
+anonymous_client = CosS3Client(anonymous_conf)
+
 client = CosS3Client(conf, retry=3)
 meta_insight_client = MetaInsightClient(metaConf, retry=3)
 ai_recognition_client = AIRecognitionClient(conf, retry=3)
@@ -6606,6 +6609,61 @@ def test_cos_client_retry_2():
     do_retry_test(client2, err_retry_bucket, 'shutdown', 1, False) 
 
 
+def test_head_exception():
+    """正确解析head请求响应的非404错误码"""
+    try:
+        anonymous_client.head_bucket(Bucket=test_bucket)
+    except CosServiceError as e:
+        assert 'x-cos-request-id' in e.get_digest_msg()
+        assert 403 == e.get_status_code()
+
+    try:
+        anonymous_client.head_object(Bucket=test_bucket, Key='foobar')
+    except CosServiceError as e:
+        assert 'x-cos-request-id' in e.get_digest_msg()
+        assert 403 == e.get_status_code()
+
+
+def test_put_object_with_tagging():
+    """上传对象时设置标签"""
+    key = 'tagging.obj'
+    # 设置单个tag: key=value
+    client.put_object(Bucket=test_bucket, Key=key, Body=b'hello', Tagging='A=B')
+    resp = client.get_object_tagging(Bucket=test_bucket, Key=key)
+    print(resp)
+    tag = resp['TagSet']['Tag']
+    assert tag[0]['Key'] == 'A'
+    assert tag[0]['Value'] == 'B'
+    resp = client.delete_object(Bucket=test_bucket, Key=key)
+    # 设置多个tag: key1=value1&key2=value2
+    client.put_object(Bucket=test_bucket, Key=key, Body=b'hello', Tagging='tagKey1=tagValue1&tagKey2=tagValue2')
+    resp = client.get_object_tagging(Bucket=test_bucket, Key=key)
+    print(resp)
+    tag = resp['TagSet']['Tag']
+    assert tag[0]['Key'] == 'tagKey1'
+    assert tag[0]['Value'] == 'tagValue1'
+    assert tag[1]['Key'] == 'tagKey2'
+    assert tag[1]['Value'] == 'tagValue2'
+    resp = client.delete_object(Bucket=test_bucket, Key=key)
+
+    # 高级接口upload_file
+    filename = 'BIG_20M'
+    gen_file(filename, 20)
+
+    resp = client.upload_file(Bucket=test_bucket, Key=key, LocalFilePath=filename, PartSize=1, Tagging='A=B')
+    print(resp)
+    resp = client.get_object_tagging(Bucket=test_bucket, Key=key)
+    print(resp)
+    tag = resp['TagSet']['Tag']
+    assert tag[0]['Key'] == 'A'
+    assert tag[0]['Value'] == 'B'
+    resp = client.delete_object(Bucket=test_bucket, Key=key)
+
+    if os.path.exists(filename):
+        os.remove(filename)
+    
+
+
 if __name__ == "__main__":
     setUp()
     """