Merge pull request #204 from zhy1985555/master

支持新域名、静态连接池

Author: zhy1985555

demo/ci_media.py

@@ -90,7 +90,7 @@ def ci_create_media_transcode_watermark_jobs():
                 'StartTime':'0',
                 'EndTime':'1000.5',
                 'Image': {
-                    'Url': 'http://'+bucket_name+".cos."+region+".myqcloud.com/1215shuiyin.jpg",
+                    'Url': 'http://'+bucket_name+".cos."+region+".tencentcos.cn/1215shuiyin.jpg",
                     'Mode': 'Fixed',
                     'Width': '128',
                     'Height': '128',

demo/ci_watermark.py

@@ -24,7 +24,7 @@
 
 bucket_name = 'examplebucket-1250000000'
 # 添加盲水印
-watermark_url = 'http://{bucket}.cos.{region}.myqcloud.com/watermark.png'.format(bucket=bucket_name, region=region)
+watermark_url = 'http://{bucket}.cos.{region}.tencentcos.cn/watermark.png'.format(bucket=bucket_name, region=region)
 watermark_url_base64 = bytes.decode(base64.b64encode(str.encode(watermark_url)))
 print(watermark_url_base64)
 response, data = client.ci_put_object_from_local_file(
@@ -39,10 +39,10 @@
 print(data['ProcessResults']['Object']['ETag'])
 
 # 下载时添加盲水印
-# download_url = http://examplebucket-1250000000.cos.ap-shanghai.myqcloud.com/sample.jpeg?watermark/3/type/3/text/watermark_url_base64
+# download_url = http://examplebucket-1250000000.cos.ap-shanghai.tencentcos.cn/sample.jpeg?watermark/3/type/3/text/watermark_url_base64
 
 # 提取盲水印
-sample_url = 'http://{bucket}.cos.{region}.myqcloud.com/sample.png'.format(bucket=bucket_name, region=region)
+sample_url = 'http://{bucket}.cos.{region}.tencentcos.cn/sample.png'.format(bucket=bucket_name, region=region)
 sample_url_base64 = bytes.decode(base64.b64encode(str.encode(sample_url)))
 response, data = client.ci_put_object_from_local_file(
     Bucket=bucket_name,

demo/demo.py

@@ -38,7 +38,7 @@ def percentage(consumed_bytes, total_bytes):
     token = None               # 如果使用永久密钥不需要填入token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见https://cloud.tencent.com/document/product/436/14048
     domain = None # domain可以不填，此时使用COS区域域名访问存储桶。domain也可以填写用户自定义域名，或者桶的全球加速域名
                   # 填写用户自定义域名，比如user-define.example.com，需要先开启桶的自定义域名，具体请参见https://cloud.tencent.com/document/product/436/36638
-                  # 填写桶的全球加速域名，比如examplebucket-1250000000.cos.accelerate.myqcloud.com，需要先开启桶的全球加速功能，请参见https://cloud.tencent.com/document/product/436/38864
+                  # 填写桶的全球加速域名，比如examplebucket-1250000000.cos.accelerate.tencentcos.cn，需要先开启桶的全球加速功能，请参见https://cloud.tencent.com/document/product/436/38864
 
     config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token, Domain=domain)  # 获取配置对象
     client = CosS3Client(config)

demo/fetch_demo.py

@@ -25,7 +25,7 @@
 response = client.put_async_fetch_task(
     Bucket=test_bucket,
     FetchTaskConfiguration={
-        'Url': 'http://examplebucket-1250000000.cos.ap-beijing.myqcloud.com/exampleobject',
+        'Url': 'http://examplebucket-1250000000.cos.ap-beijing.tencentcos.cn/exampleobject',
         'Key': 'exampleobject'
     }
 )

qcloud_cos/cos_client.py

@@ -39,7 +39,7 @@ class CosConfig(object):
     def __init__(self, Appid=None, Region=None, SecretId=None, SecretKey=None, Token=None, Scheme=None, Timeout=None,
                  Access_id=None, Access_key=None, Secret_id=None, Secret_key=None, Endpoint=None, IP=None, Port=None,
                  Anonymous=None, UA=None, Proxies=None, Domain=None, ServiceDomain=None, PoolConnections=10,
-                 PoolMaxSize=10, AllowRedirects=False, SignHost=True, EndpointCi=None):
+                 PoolMaxSize=10, AllowRedirects=False, SignHost=True, EndpointCi=None, EnableOldDomain=True, EnableInternalDomain=True):
         """初始化，保存用户的信息
 
         :param Appid(string): 用户APPID.
@@ -66,6 +66,8 @@ def __init__(self, Appid=None, Region=None, SecretId=None, SecretKey=None, Token
         :param AllowRedirects(bool):  是否重定向
         :param SignHost(bool):  是否将host算入签名
         :param EndpointCi(string):  ci的endpoint
+        :param EnableOldDomain(bool):  是否使用旧的myqcloud.com域名访问COS
+        :param EnableInternalDomain(bool):  是否使用内网域名访问COS
         """
         self._appid = to_unicode(Appid)
         self._token = to_unicode(Token)
@@ -85,9 +87,11 @@ def __init__(self, Appid=None, Region=None, SecretId=None, SecretKey=None, Token
         self._allow_redirects = AllowRedirects
         self._sign_host = SignHost
         self._copy_part_threshold_size = SINGLE_UPLOAD_LENGTH
+        self._enable_old_domain = EnableOldDomain
+        self._enable_internal_domain = EnableInternalDomain
 
         if self._domain is None:
-            self._endpoint = format_endpoint(Endpoint, Region)
+            self._endpoint = format_endpoint(Endpoint, Region, u'cos.', EnableOldDomain, EnableInternalDomain)
         if Scheme is None:
             Scheme = u'https'
         Scheme = to_unicode(Scheme)
@@ -96,7 +100,8 @@ def __init__(self, Appid=None, Region=None, SecretId=None, SecretKey=None, Token
         self._scheme = Scheme
 
         # 格式化ci的endpoint 不支持自定义域名的
-        self._endpoint_ci = format_endpoint(EndpointCi, Region, u'ci.')
+        # ci暂不支持新域名
+        self._endpoint_ci = format_endpoint(EndpointCi, Region, u'ci.', True, False)
 
         # 兼容(SecretId,SecretKey)以及(AccessId,AccessKey)
         if (SecretId and SecretKey):
@@ -205,6 +210,8 @@ def convert_secret_value(self, value):
 class CosS3Client(object):
     """cos客户端类，封装相应请求"""
 
+    __built_in_sessions = None  # 内置的静态连接池，多个Client间共享使用
+
     def __init__(self, conf, retry=1, session=None):
         """初始化client对象
 
@@ -214,15 +221,48 @@ def __init__(self, conf, retry=1, session=None):
         """
         self._conf = conf
         self._retry = retry  # 重试的次数，分片上传时可适当增大
+
+        if not CosS3Client.__built_in_sessions:
+            with threading.Lock():
+                if not CosS3Client.__built_in_sessions:  # 加锁后double check
+                    CosS3Client.__built_in_sessions = self.generate_built_in_connection_pool(self._conf._pool_connections, self._conf._pool_maxsize)
+
         if session is None:
-            self._session = requests.session()
-            self._session.mount('http://', requests.adapters.HTTPAdapter(pool_connections=self._conf._pool_connections,
-                                                                         pool_maxsize=self._conf._pool_maxsize))
-            self._session.mount('https://', requests.adapters.HTTPAdapter(pool_connections=self._conf._pool_connections,
-                                                                          pool_maxsize=self._conf._pool_maxsize))
+            self._session = CosS3Client.__built_in_sessions
         else:
             self._session = session
 
+    def set_built_in_connection_pool_max_size(self, PoolConnections, PoolMaxSize):
+        """设置SDK内置的连接池的连接大小，并且重新绑定到client中"""
+        if not CosS3Client.__built_in_sessions:
+            return
+
+        if CosS3Client.__built_in_sessions.get_adapter('http://')._pool_connections == PoolConnections \
+           and CosS3Client.__built_in_sessions.get_adapter('http://')._pool_maxsize == PoolMaxSize:
+            return
+
+        # 判断之前是否绑定到内置连接池
+        rebound = False
+        if self._session and self._session is CosS3Client.__built_in_sessions:
+            rebound = True
+
+        # 重新生成内置连接池
+        CosS3Client.__built_in_sessions.close()
+        CosS3Client.__built_in_sessions = self.generate_built_in_connection_pool(PoolConnections, PoolMaxSize)
+
+        # 重新绑定到内置连接池
+        if rebound:
+            self._session = CosS3Client.__built_in_sessions
+            logger.warn("rebound built-in connection pool success. maxsize=%d,%d" % (PoolConnections, PoolMaxSize))
+
+    def generate_built_in_connection_pool(self, PoolConnections, PoolMaxSize):
+        """生成SDK内置的连接池，此连接池是client间共用的"""
+        built_in_sessions = requests.session()
+        built_in_sessions.mount('http://', requests.adapters.HTTPAdapter(pool_connections=PoolConnections, pool_maxsize=PoolMaxSize))
+        built_in_sessions.mount('https://', requests.adapters.HTTPAdapter(pool_connections=PoolConnections, pool_maxsize=PoolMaxSize))
+        logger.warn("generate built-in connection pool success. maxsize=%d,%d" % (PoolConnections, PoolMaxSize))
+        return built_in_sessions
+
     def get_conf(self):
         """获取配置"""
         return self._conf
@@ -778,7 +818,7 @@ def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs):
             )
         """
         headers = mapped(kwargs)
-        headers['x-cos-copy-source'] = gen_copy_source_url(CopySource)
+        headers['x-cos-copy-source'] = gen_copy_source_url(CopySource, self._conf._enable_old_domain, self._conf._enable_internal_domain)
         if CopyStatus != 'Copy' and CopyStatus != 'Replaced':
             raise CosClientError('CopyStatus must be Copy or Replaced')
         headers['x-cos-metadata-directive'] = CopyStatus
@@ -827,7 +867,7 @@ def upload_part_copy(self, Bucket, Key, PartNumber, UploadId, CopySource, CopySo
             )
         """
         headers = mapped(kwargs)
-        headers['x-cos-copy-source'] = gen_copy_source_url(CopySource)
+        headers['x-cos-copy-source'] = gen_copy_source_url(CopySource, self._conf._enable_old_domain, self._conf._enable_internal_domain)
         headers['x-cos-copy-source-range'] = CopySourceRange
         params = {'partNumber': PartNumber, 'uploadId': UploadId}
         params = format_values(params)
@@ -3199,7 +3239,12 @@ def list_buckets(self, **kwargs):
             response = client.list_buckets()
         """
         headers = mapped(kwargs)
-        url = '{scheme}://service.cos.myqcloud.com/'.format(scheme=self._conf._scheme)
+
+        if self._conf._enable_old_domain:
+            url = '{scheme}://service.cos.myqcloud.com/'.format(scheme=self._conf._scheme)
+        else:
+            url = '{scheme}://service.cos.tencentcos.cn/'.format(scheme=self._conf._scheme)
+
         if self._conf._service_domain is not None:
             url = '{scheme}://{domain}/'.format(scheme=self._conf._scheme, domain=self._conf._service_domain)
         rt = self.send_request(
@@ -3476,7 +3521,7 @@ def upload_file(self, Bucket, Key, LocalFilePath, PartSize=1, MAXThread=5, Enabl
 
     def _head_object_when_copy(self, CopySource, **kwargs):
         """查询源文件的长度"""
-        bucket, path, endpoint, versionid = get_copy_source_info(CopySource)
+        bucket, path, endpoint, versionid = get_copy_source_info(CopySource, self._conf._enable_old_domain, self._conf._enable_internal_domain)
         params = {}
         if versionid != '':
             params['versionId'] = versionid
@@ -3521,7 +3566,7 @@ def _upload_part_copy(self, bucket, key, part_number, upload_id, copy_source, co
         return None
 
     def _check_same_region(self, dst_endpoint, CopySource):
-        src_endpoint = get_copy_source_info(CopySource)[2]
+        src_endpoint = get_copy_source_info(CopySource, self._conf._enable_old_domain, self._conf._enable_internal_domain)[2]
         if src_endpoint == dst_endpoint:
             return True
         return False
@@ -5244,7 +5289,7 @@ def ci_auditing_image_batch(self, Bucket, Input, DetectType=None, BizType=None,
         :param Input(dict array): 需要审核的图片信息,每个array元素为dict类型，支持的参数如下:
                             Object: 存储在 COS 存储桶中的图片文件名称，例如在目录 test 中的文件 image.jpg，则文件名称为 test/image.jpg。
                                 Object 和 Url 只能选择其中一种。
-                            Url: 图片文件的链接地址，例如 http://a-1250000.cos.ap-shanghai.myqcloud.com/image.jpg。
+                            Url: 图片文件的链接地址，例如 http://a-1250000.cos.ap-shanghai.tencentcos.cn/image.jpg。
                                 Object 和 Url 只能选择其中一种。
                             Interval: 截帧频率，GIF 图检测专用，默认值为5，表示从第一帧（包含）开始每隔5帧截取一帧
                             MaxFrames: 最大截帧数量，GIF 图检测专用，默认值为5，表示只截取 GIF 的5帧图片进行审核，必须大于0

qcloud_cos/cos_comm.py

@@ -220,7 +220,7 @@ def format_values(data):
     return data
 
 
-def format_endpoint(endpoint, region, module=u'cos.'):
+def format_endpoint(endpoint, region, module, EnableOldDomain, EnableInternalDomain):
     # 客户使用全球加速域名时，只会传endpoint不会传region。此时这样endpointCi和region同时为None，就会报错。
     if not endpoint and not region and module == u'cos.':
         raise CosClientError("Region or Endpoint is required not empty!")
@@ -229,13 +229,16 @@ def format_endpoint(endpoint, region, module=u'cos.'):
     if endpoint:
         return to_unicode(endpoint)
     elif region:
-        region = format_region(region, module)
-        return u"{region}.myqcloud.com".format(region=region)
+        region = format_region(region, module, EnableOldDomain, EnableInternalDomain)
+        if EnableOldDomain:
+            return u"{region}.myqcloud.com".format(region=region)
+        else:
+            return u"{region}.tencentcos.cn".format(region=region)
     else:
         return None
 
 
-def format_region(region, module=u'cos.'):
+def format_region(region, module, EnableOldDomain, EnableInternalDomain):
     """格式化地域"""
     if not isinstance(region, string_types):
         raise CosClientError("region is not string type")
@@ -249,24 +252,29 @@ def format_region(region, module=u'cos.'):
     if region == u'cn-north' or region == u'cn-south' or region == u'cn-east' or region == u'cn-south-2' or region == u'cn-southwest' or region == u'sg':
         return region  # 老域名不能加cos.
     #  支持v4域名映射到v5
+
+    # 转换为内部域名 (只有新域名才支持内部域名)
+    if not EnableOldDomain and EnableInternalDomain and module == u'cos.':
+        module = u'cos-internal.'
+
     if region == u'cossh':
-        return u'cos.ap-shanghai'
+        return module + u'ap-shanghai'
     if region == u'cosgz':
-        return u'cos.ap-guangzhou'
+        return module + u'ap-guangzhou'
     if region == 'cosbj':
-        return u'cos.ap-beijing'
+        return module + u'ap-beijing'
     if region == 'costj':
-        return u'cos.ap-beijing-1'
+        return module + u'ap-beijing-1'
     if region == u'coscd':
-        return u'cos.ap-chengdu'
+        return module + u'ap-chengdu'
     if region == u'cossgp':
-        return u'cos.ap-singapore'
+        return module + u'ap-singapore'
     if region == u'coshk':
-        return u'cos.ap-hongkong'
+        return module + u'ap-hongkong'
     if region == u'cosca':
-        return u'cos.na-toronto'
+        return module + u'na-toronto'
     if region == u'cosger':
-        return u'cos.eu-frankfurt'
+        return module + u'eu-frankfurt'
 
     return module + region  # 新域名加上cos.
 
@@ -306,7 +314,7 @@ def format_path(path):
     return path
 
 
-def get_copy_source_info(CopySource):
+def get_copy_source_info(CopySource, EnableOldDomain, EnableInternalDomain):
     """获取拷贝源的所有信息"""
     appid = u""
     versionid = u""
@@ -323,7 +331,7 @@ def get_copy_source_info(CopySource):
         region = CopySource['Region']
     if 'Endpoint' in CopySource:
         endpoint = CopySource['Endpoint']
-    endpoint = format_endpoint(endpoint, region)
+    endpoint = format_endpoint(endpoint, region, u'cos.', EnableOldDomain, EnableInternalDomain)
     if 'Key' in CopySource:
         path = to_unicode(CopySource['Key'])
         if path and path[0] == '/':
@@ -335,9 +343,9 @@ def get_copy_source_info(CopySource):
     return bucket, path, endpoint, versionid
 
 
-def gen_copy_source_url(CopySource):
+def gen_copy_source_url(CopySource, EnableOldDomain, EnableInternalDomain):
     """拼接拷贝源url"""
-    bucket, path, endpoint, versionid = get_copy_source_info(CopySource)
+    bucket, path, endpoint, versionid = get_copy_source_info(CopySource, EnableOldDomain, EnableInternalDomain)
     path = format_path(path)
     if versionid != u'':
         path = path + u'?versionId=' + versionid

qcloud_cos/cos_threadpool.py

@@ -32,7 +32,12 @@ def run(self):
             except Exception as e:
                 logger.warn(str(e))
                 self._fail_task_num += 1
-                self._ret.append(e)
+                if hasattr(e, '_message') and e._message:
+                    self._ret.append(e._message)
+                elif hasattr(e, 'message') and e.message:
+                    self._ret.append(e.message)
+                else:
+                    self._ret.append('meet some exception')
             finally:
                 self._task_queue.task_done()
 

qcloud_cos/version.py

@@ -1 +1 @@
-__version__ = '5.1.9.15'
+__version__ = '5.1.9.16'

setup.py

@@ -16,7 +16,7 @@ def long_description():
 
 setup(
     name='cos-python-sdk-v5',
-    version='1.9.15',
+    version='1.9.16',
     url='https://www.qcloud.com/',
     license='MIT',
     author='tiedu, lewzylu, channingliu',

ut/test.py

@@ -788,7 +788,10 @@ def test_use_get_auth():
         Key='test.txt',
         Params={'acl': '', 'unsed': '123'}
     )
-    url = 'http://' + test_bucket + '.cos.' + REGION + '.myqcloud.com/test.txt?acl&unsed=123'
+    if conf._enable_old_domain:
+        url = 'http://' + test_bucket + '.cos.' + REGION + '.myqcloud.com/test.txt?acl&unsed=123'
+    else:
+        url = 'http://' + test_bucket + '.cos.' + REGION + '.tencentcos.cn/test.txt?acl&unsed=123'
     response = requests.get(url, headers={'Authorization': auth})
     assert response.status_code == 200
 
@@ -1681,7 +1684,7 @@ def test_ci_create_media_transcode_watermark_jobs():
                     'StartTime': '0',
                     'EndTime': '1000.5',
                     'Image': {
-                        'Url': 'http://'+ci_bucket_name+".cos."+ci_region+".myqcloud.com/1215shuiyin.jpg",
+                        'Url': 'http://'+ci_bucket_name+".cos."+ci_region+".tencentcos.cn/1215shuiyin.jpg",
                         'Mode': 'Fixed',
                         'Width': '128',
                         'Height': '128',