From c34da55c37532f5175652d122035201ce5d94d5d Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Fri, 10 May 2019 09:57:05 -0500 Subject: [PATCH 1/8] applying backoff for rate-limited Maven downloads --- setup.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 99aba8bf..90d8be50 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,8 @@ import os import shutil +from time import sleep + from setuptools import Command from setuptools import setup from setuptools.command.install import install @@ -121,6 +123,7 @@ ('commons-beanutils', 'commons-beanutils', '1.9.3'), ('commons-collections', 'commons-collections', '3.2.2') ] +MAX_URL_DOWNLOAD_ATTEMPTS = 5 class MavenJarDownloader: @@ -181,7 +184,8 @@ def download_file(self, url, dest): """ print('Attempting to retrieve remote jar {url}'.format(url=url)) try: - response = urlopen(url) + response = self.make_request_with_backoff(url) + with open(dest, 'wb') as dest_file: shutil.copyfileobj(response, dest_file) print('Saving {url} -> {dest}'.format(url=url, dest=dest)) @@ -198,6 +202,18 @@ def download_files(self): url = self.package_url(package[0], package[1], package[2]) self.download_file(url, dest) + def make_request_with_backoff(self, url): + for attempt_number in range(MAX_URL_DOWNLOAD_ATTEMPTS): + response = urlopen(url) + breakpoint() + if response.getcode() == 429: + sleep_time = 2 ** attempt_number + print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time)) + sleep(sleep_time) + else: + return response + raise Exception('"429 Too Many Requests" responses received.') + class DownloadJarsCommand(Command): description = "Download the jar files needed to run the sample application" From 8d4eab3a664cdd6dfcde1d309b0f6d4771e2dcf9 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Fri, 10 May 2019 10:14:07 -0500 Subject: [PATCH 2/8] removing breakpoint --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 90d8be50..1f453ad2 100644 --- a/setup.py +++ b/setup.py @@ -205,7 +205,6 @@ def download_files(self): def make_request_with_backoff(self, url): for attempt_number in range(MAX_URL_DOWNLOAD_ATTEMPTS): response = urlopen(url) - breakpoint() if response.getcode() == 429: sleep_time = 2 ** attempt_number print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time)) From bd210b1e083509081d09d02cde883c043d585919 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Wed, 22 May 2019 13:24:40 -0500 Subject: [PATCH 3/8] incrementing version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f453ad2..32d6555e 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ PACKAGE_NAME = 'amazon_kclpy' JAR_DIRECTORY = os.path.join(PACKAGE_NAME, 'jars') -PACKAGE_VERSION = '2.0.1' +PACKAGE_VERSION = '2.0.2' PYTHON_REQUIREMENTS = [ 'boto', # argparse is part of python2.7 but must be declared for python2.6 From 4ba4d9e39ecdcc8c3e62487fdc4e20eb1ebc0a9c Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Wed, 22 May 2019 14:01:27 -0500 Subject: [PATCH 4/8] moving print statement to correct place --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 32d6555e..827f541b 100644 --- a/setup.py +++ b/setup.py @@ -182,7 +182,6 @@ def download_file(self, url, dest): """ Downloads a file at the url to the destination. """ - print('Attempting to retrieve remote jar {url}'.format(url=url)) try: response = self.make_request_with_backoff(url) @@ -201,9 +200,11 @@ def download_files(self): else: url = self.package_url(package[0], package[1], package[2]) self.download_file(url, dest) + raise Exception() def make_request_with_backoff(self, url): for attempt_number in range(MAX_URL_DOWNLOAD_ATTEMPTS): + print('Attempting to retrieve remote jar {url}'.format(url=url)) response = urlopen(url) if response.getcode() == 429: sleep_time = 2 ** attempt_number From 9ea355462cae27dc986154b7095a7b8ff068edd0 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Wed, 22 May 2019 14:02:27 -0500 Subject: [PATCH 5/8] incrementing version --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 827f541b..97d797a8 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ PACKAGE_NAME = 'amazon_kclpy' JAR_DIRECTORY = os.path.join(PACKAGE_NAME, 'jars') -PACKAGE_VERSION = '2.0.2' +PACKAGE_VERSION = '2.0.3' PYTHON_REQUIREMENTS = [ 'boto', # argparse is part of python2.7 but must be declared for python2.6 @@ -200,7 +200,6 @@ def download_files(self): else: url = self.package_url(package[0], package[1], package[2]) self.download_file(url, dest) - raise Exception() def make_request_with_backoff(self, url): for attempt_number in range(MAX_URL_DOWNLOAD_ATTEMPTS): From 5c3f4a0a1f14642b61c9f7ea1e567dd6328c9c62 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Wed, 22 May 2019 14:31:34 -0500 Subject: [PATCH 6/8] making backoff catch the code correctly --- setup.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 97d797a8..77240baa 100644 --- a/setup.py +++ b/setup.py @@ -26,9 +26,11 @@ if sys.version_info[0] >= 3: # Python 3 from urllib.request import urlopen + from urllib.error import HTTPError else: # Python 2 from urllib2 import urlopen + from urllib2 import HTTPError # # This script modifies the basic setuptools by adding some functionality to the standard @@ -52,7 +54,7 @@ PACKAGE_NAME = 'amazon_kclpy' JAR_DIRECTORY = os.path.join(PACKAGE_NAME, 'jars') -PACKAGE_VERSION = '2.0.3' +PACKAGE_VERSION = '2.0.4' PYTHON_REQUIREMENTS = [ 'boto', # argparse is part of python2.7 but must be declared for python2.6 @@ -204,8 +206,10 @@ def download_files(self): def make_request_with_backoff(self, url): for attempt_number in range(MAX_URL_DOWNLOAD_ATTEMPTS): print('Attempting to retrieve remote jar {url}'.format(url=url)) - response = urlopen(url) - if response.getcode() == 429: + try: + return urlopen(url) + except HTTPError as e: + if e.code == 429: sleep_time = 2 ** attempt_number print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time)) sleep(sleep_time) From 9ce45234ce4e361f0d0db0d0f95d903e293e73d8 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Wed, 22 May 2019 14:42:23 -0500 Subject: [PATCH 7/8] fixing indentation level --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 77240baa..b44dac41 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ PACKAGE_NAME = 'amazon_kclpy' JAR_DIRECTORY = os.path.join(PACKAGE_NAME, 'jars') -PACKAGE_VERSION = '2.0.4' +PACKAGE_VERSION = '2.0.5' PYTHON_REQUIREMENTS = [ 'boto', # argparse is part of python2.7 but must be declared for python2.6 @@ -209,10 +209,10 @@ def make_request_with_backoff(self, url): try: return urlopen(url) except HTTPError as e: - if e.code == 429: - sleep_time = 2 ** attempt_number - print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time)) - sleep(sleep_time) + if e.code == 429: + sleep_time = 2 ** attempt_number + print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time)) + sleep(sleep_time) else: return response raise Exception('"429 Too Many Requests" responses received.') From c78c9dbc843b72337bd88a8a5b48362ba88b78c9 Mon Sep 17 00:00:00 2001 From: Jon Falkenstein Date: Tue, 4 Jun 2019 15:44:00 -0500 Subject: [PATCH 8/8] fixing error handling in setup.py --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b44dac41..67ef3057 100644 --- a/setup.py +++ b/setup.py @@ -213,8 +213,9 @@ def make_request_with_backoff(self, url): sleep_time = 2 ** attempt_number print('"429 Too Many Requests" response received. Sleeping {} seconds and trying again.'.format(sleep_time)) sleep(sleep_time) - else: - return response + else: + raise + raise Exception('"429 Too Many Requests" responses received.')