Merge pull request #121 from dt3310321/s3

S3

Author: dt3310321

.travis.yml

@@ -16,7 +16,7 @@ notifications:
       - wjielai@tencent.com
       - fysntian@tencent.com
 script:
-- pycodestyle --max-line-length=180 qcloud_cos/.
+- pycodestyle --max-line-length=200 qcloud_cos/.
 - nosetests -s -v ut/
 deploy:
   provider: pypi

qcloud_cos/cos_auth.py

@@ -6,7 +6,7 @@
 import hashlib
 import logging
 from requests.auth import AuthBase
-from .cos_comm import to_unicode, to_bytes
+from .cos_comm import to_unicode, to_bytes, to_str
 logger = logging.getLogger(__name__)
 
 
@@ -44,14 +44,13 @@ def __call__(self, r):
         path = self._path
         uri_params = self._params
         headers = filter_headers(r.headers)
-        uri_params = dict([(k.lower(), v) for k, v in uri_params.items()])
         # reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+)
-        headers = dict([(k.lower(), quote(to_bytes(v), '-_.~')) for k, v in headers.items()])  # headers中的key转换为小写，value进行encode
-        uri_params = dict([(k.lower(), v) for k, v in uri_params.items()])
+        headers = dict([(quote(to_bytes(to_str(k)), '-_.~').lower(), quote(to_bytes(to_str(v)), '-_.~')) for k, v in headers.items()])  # headers中的key转换为小写，value进行encode
+        uri_params = dict([(quote(to_bytes(to_str(k)), '-_.~').lower(), quote(to_bytes(to_str(v)), '-_.~')) for k, v in uri_params.items()])
         format_str = u"{method}\n{host}\n{params}\n{headers}\n".format(
             method=r.method.lower(),
             host=path,
-            params=urlencode(sorted(uri_params.items())).replace('+', '%20').replace('%7E', '~'),
+            params='&'.join(map(lambda tupl: "%s=%s" % (tupl[0], tupl[1]), sorted(uri_params.items()))),
             headers='&'.join(map(lambda tupl: "%s=%s" % (tupl[0], tupl[1]), sorted(headers.items())))
         )
         logger.debug("format str: " + format_str)
@@ -73,7 +72,7 @@ def __call__(self, r):
             ak=self._secret_id,
             sign_time=sign_time,
             key_time=sign_time,
-            params=';'.join(sorted(map(lambda k: k.lower(), uri_params.keys()))),
+            params=';'.join(sorted(uri_params.keys())),
             headers=';'.join(sorted(headers.keys())),
             sign=sign
         )

qcloud_cos/cos_client.py

@@ -479,6 +479,7 @@ def get_presigned_url(self, Bucket, Key, Method, Expired=300, Params={}, Headers
         """
         url = self._conf.uri(bucket=Bucket, path=Key)
         sign = self.get_auth(Method=Method, Bucket=Bucket, Key=Key, Expired=Expired, Headers=Headers, Params=Params)
+        sign = urlencode(dict([item.split('=', 1) for item in sign.split('&')]))
         url = url + '?' + sign
         if Params:
             url = url + '&' + urlencode(Params)
@@ -2828,7 +2829,7 @@ def list_buckets(self, **kwargs):
         return data
 
     # Advanced interface
-    def _upload_part(self, bucket, key, local_path, offset, size, part_num, uploadid, md5_lst, resumable_flag, already_exist_parts, enable_md5):
+    def _upload_part(self, bucket, key, local_path, offset, size, part_num, uploadid, md5_lst, resumable_flag, already_exist_parts, enable_md5, traffic_limit):
         """从本地文件中读取分块, 上传单个分块,将结果记录在md5——list中
 
         :param bucket(string): 存储桶名称.
@@ -2851,7 +2852,7 @@ def _upload_part(self, bucket, key, local_path, offset, size, part_num, uploadid
             with open(local_path, 'rb') as fp:
                 fp.seek(offset, 0)
                 data = fp.read(size)
-            rt = self.upload_part(bucket, key, data, part_num, uploadid, enable_md5)
+            rt = self.upload_part(bucket, key, data, part_num, uploadid, enable_md5, TrafficLimit=traffic_limit)
             md5_lst.append({'PartNumber': part_num, 'ETag': rt['ETag']})
         return None
 
@@ -3004,15 +3005,19 @@ def upload_file(self, Bucket, Key, LocalFilePath, PartSize=1, MAXThread=5, Enabl
                 logger.info("create a new uploadid in upload_file, uploadid={uploadid}".format(uploadid=uploadid))
 
             # 上传分块
+            # 增加限速功能
+            traffic_limit = None
+            if 'TrafficLimit' in kwargs:
+                traffic_limit = kwargs['TrafficLimit']
             offset = 0  # 记录文件偏移量
             lst = list()  # 记录分块信息
             pool = SimpleThreadPool(MAXThread)
 
             for i in range(1, parts_num+1):
                 if i == parts_num:  # 最后一块
-                    pool.add_task(self._upload_part, Bucket, Key, LocalFilePath, offset, file_size-offset, i, uploadid, lst, resumable_flag, already_exist_parts, EnableMD5)
+                    pool.add_task(self._upload_part, Bucket, Key, LocalFilePath, offset, file_size-offset, i, uploadid, lst, resumable_flag, already_exist_parts, EnableMD5, traffic_limit)
                 else:
-                    pool.add_task(self._upload_part, Bucket, Key, LocalFilePath, offset, part_size, i, uploadid, lst, resumable_flag, already_exist_parts, EnableMD5)
+                    pool.add_task(self._upload_part, Bucket, Key, LocalFilePath, offset, part_size, i, uploadid, lst, resumable_flag, already_exist_parts, EnableMD5, traffic_limit)
                     offset += part_size
 
             pool.wait_completion()

qcloud_cos/cos_comm.py

@@ -61,6 +61,13 @@
            }
 
 
+def to_str(s):
+    """非字符串转换为字符串"""
+    if isinstance(s, text_type) or isinstance(s, binary_type):
+        return s
+    return str(s)
+
+
 def to_unicode(s):
     """将字符串转为unicode"""
     if isinstance(s, binary_type):

qcloud_cos/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '5.1.8.0'
+__version__ = '5.1.8.1'

setup.py

@@ -16,7 +16,7 @@ def long_description():
 
 setup(
     name='cos-python-sdk-v5',
-    version='1.8.0',
+    version='1.8.1',
     url='https://www.qcloud.com/',
     license='MIT',
     author='tiedu, lewzylu, channingliu',