Merge pull request #259 from l-iberty/master

重试时domain切换逻辑修改

Author: l-iberty

qcloud_cos/cos_client.py

@@ -43,7 +43,8 @@ class CosConfig(object):
     def __init__(self, Appid=None, Region=None, SecretId=None, SecretKey=None, Token=None, CredentialInstance=None, Scheme=None, Timeout=None,
                  Access_id=None, Access_key=None, Secret_id=None, Secret_key=None, Endpoint=None, IP=None, Port=None,
                  Anonymous=None, UA=None, Proxies=None, Domain=None, ServiceDomain=None, KeepAlive=True, PoolConnections=10,
-                 PoolMaxSize=10, AllowRedirects=False, SignHost=True, EndpointCi=None, EndpointPic=None, EnableOldDomain=True, EnableInternalDomain=True, SignParams=True):
+                 PoolMaxSize=10, AllowRedirects=False, SignHost=True, EndpointCi=None, EndpointPic=None, EnableOldDomain=True, EnableInternalDomain=True, SignParams=True,
+                 AutoSwitchDomainOnRetry=True):
         """初始化，保存用户的信息
 
         :param Appid(string): 用户APPID.
@@ -74,6 +75,7 @@ def __init__(self, Appid=None, Region=None, SecretId=None, SecretKey=None, Token
         :param EnableOldDomain(bool):  是否使用旧的myqcloud.com域名访问COS
         :param EnableInternalDomain(bool):  是否使用内网域名访问COS
         :param SignParams(bool): 是否将请求参数算入签名
+        :param AutoSwitchDomainOnRetry(bool): 重试请求时是否自动切换域名
         """
         self._appid = to_unicode(Appid)
         self._token = to_unicode(Token)
@@ -98,6 +100,7 @@ def __init__(self, Appid=None, Region=None, SecretId=None, SecretKey=None, Token
         self._enable_old_domain = EnableOldDomain
         self._enable_internal_domain = EnableInternalDomain
         self._sign_params = SignParams
+        self._auto_switch_domain_on_retry = AutoSwitchDomainOnRetry
 
         if self._domain is None:
             self._endpoint = format_endpoint(Endpoint, Region, u'cos.', EnableOldDomain, EnableInternalDomain)
@@ -347,6 +350,7 @@ def send_request(self, method, url, bucket, timeout=30, cos_request=True, ci_req
         kwargs['headers'] = format_values(kwargs['headers'])
 
         file_position = None
+        domain_switched = False # 重试时如果要切换域名, 只切换一次
         if 'data' in kwargs:
             body = kwargs['data']
             if hasattr(body, 'tell') and hasattr(body, 'seek') and hasattr(body, 'read'):
@@ -376,13 +380,25 @@ def send_request(self, method, url, bucket, timeout=30, cos_request=True, ci_req
                     break
                 else:
                     if j < self._retry and client_can_retry(file_position, **kwargs):
+                        if not domain_switched and self._conf._auto_switch_domain_on_retry and self._conf._ip is None:
+                            # 重试时切换域名
+                            logger.debug("switch hostname, url before: " + url)
+                            url = switch_hostname_for_url(url)
+                            domain_switched = True
+                            logger.debug("switch hostname, url after: " + url)
                         continue
                     else:
                         break
             except Exception as e:  # 捕获requests抛出的如timeout等客户端错误,转化为客户端错误
                 logger.exception('url:%s, retry_time:%d exception:%s' % (url, j, str(e)))
                 if j < self._retry and (isinstance(e, ConnectionError) or isinstance(e, Timeout)):  # 只重试网络错误
                     if client_can_retry(file_position, **kwargs):
+                        if not domain_switched and self._conf._auto_switch_domain_on_retry and self._conf._ip is None:
+                            # 重试时切换域名
+                            logger.debug("switch hostname, url before: " + url)
+                            url = switch_hostname_for_url(url)
+                            domain_switched = True
+                            logger.debug("switch hostname, url after: " + url)
                         continue
                 raise CosClientError(str(e))
 
@@ -3447,7 +3463,7 @@ def _upload_part(self, bucket, key, local_path, offset, size, part_num, uploadid
                 fp.seek(offset, 0)
                 data = fp.read(size)
             rt = self.upload_part(bucket, key, data, part_num, uploadid, enable_md5, **kwargs)
-            lower_rt = { k.lower():v for k,v in rt.items() }
+            lower_rt = dict([(k.lower(), v) for k, v in rt.items()])
             md5_lst.append({'PartNumber': part_num, 'ETag': lower_rt['etag']})
         if progress_callback:
             progress_callback.report(size)

qcloud_cos/cos_comm.py

@@ -1,7 +1,7 @@
 # -*- coding=utf-8
 
 from six import text_type, binary_type, string_types
-from six.moves.urllib.parse import quote, unquote
+from six.moves.urllib.parse import quote, unquote, urlparse
 import hashlib
 import base64
 import os
@@ -244,6 +244,32 @@ def format_endpoint(endpoint, region, module, EnableOldDomain, EnableInternalDom
     else:
         return None
 
+def switch_hostname(host):
+    """将cos默认域名的.myqcloud.com后缀替换为.tencentcos.cn"""
+    if not host:
+        raise CosClientError("Host is required not empty!")
+    
+    # *.cos.*-*.myqcloud.com
+    if re.match(r'^.*\.cos\..*\-.*\.myqcloud\.com$', host):
+        host = host[:-len(".myqcloud.com")] + ".tencentcos.cn"
+    
+    return host
+
+def switch_hostname_for_url(url):
+    if not url:
+        raise CosClientError("Url is required not empty!")
+
+    url_parsed = urlparse(url)
+    if url_parsed.hostname is not None:
+        host = url_parsed.hostname
+        new_host = switch_hostname(host)
+        if host != new_host:
+            new_url = url.replace(host, new_host)
+            return new_url
+
+    """返回原始url"""
+    return url
+
 
 def format_region(region, module, EnableOldDomain, EnableInternalDomain):
     """格式化地域"""

ut/test.py

@@ -15,7 +15,7 @@
 from qcloud_cos.cos_encryption_client import CosEncryptionClient
 from qcloud_cos.crypto import AESProvider
 from qcloud_cos.crypto import RSAProvider
-from qcloud_cos.cos_comm import CiDetectType, get_md5, to_bytes
+from qcloud_cos.cos_comm import CiDetectType, get_md5, to_bytes, switch_hostname_for_url
 
 SECRET_ID = os.environ["COS_SECRET_ID"]
 SECRET_KEY = os.environ["COS_SECRET_KEY"]
@@ -3562,6 +3562,37 @@ def test_check_multipart_upload():
         UploadId=uploadId
     )
 
+def test_switch_hostname_for_url():
+    url = "https://example-125000000.cos.ap-chengdu.myqcloud.com/123"
+    res = switch_hostname_for_url(url)
+    exp = "https://example-125000000.cos.ap-chengdu.tencentcos.cn/123"
+    assert res == exp
+
+    url = "https://example-125000000.cos.ap-chengdu.tencentcos.cn/123"
+    res = switch_hostname_for_url(url)
+    exp = "https://example-125000000.cos.ap-chengdu.tencentcos.cn/123"
+    assert res == exp
+
+    url = "https://cos.ap-chengdu.myqcloud.com/123"
+    res = switch_hostname_for_url(url)
+    exp = "https://cos.ap-chengdu.myqcloud.com/123"
+    assert res == exp
+
+    url = "https://service.cos.myqcloud.com/123"
+    res = switch_hostname_for_url(url)
+    exp = "https://service.cos.myqcloud.com/123"
+    assert res == exp
+
+    url = "https://example-125000000.file.myqcloud.com/123"
+    res = switch_hostname_for_url(url)
+    exp = "https://example-125000000.file.myqcloud.com/123"
+    assert res == exp
+
+    try:
+        switch_hostname_for_url('')
+    except Exception as e:
+        print(e)
+
 if __name__ == "__main__":
     setUp()
     """