Merge pull request #73 from pageuppeople-opensource/retry_dropoff

Support exponential delay for lambda retry attempts

Author: ChintanRaval

rdl/data_sources/AWSLambdaDataSource.py

@@ -2,6 +2,7 @@
 import pandas
 import json
 import boto3
+import time
 
 from rdl.data_sources.ChangeTrackingInfo import ChangeTrackingInfo
 from rdl.data_sources.SourceTableInfo import SourceTableInfo
@@ -149,9 +150,14 @@ def __get_data_frame(self, data: [[]], column_names: []):
 
     def __invoke_lambda(self, pay_load):
         max_attempts = Constants.MAX_AWS_LAMBDA_INVOKATION_ATTEMPTS
+        retry_delay = Constants.AWS_LAMBDA_RETRY_DELAY_SECONDS
         response_payload = None
 
         for current_attempt in list(range(1, max_attempts+1, 1)):
+            if current_attempt > 1:
+                self.logger.debug(f"\nDelaying retry for {(current_attempt - 1) ^ retry_delay} seconds")
+                time.sleep((current_attempt - 1) ^ retry_delay)
+
             self.logger.debug(f"\nRequest being sent to Lambda, attempt {current_attempt} of {max_attempts}:")
             self.logger.debug(pay_load)
 

rdl/shared/Constants.py

@@ -1,6 +1,7 @@
 APP_NAME = "Relational Data Loader"
 DATA_PIPELINE_EXECUTION_SCHEMA_NAME = "rdl"
-MAX_AWS_LAMBDA_INVOKATION_ATTEMPTS = 3
+MAX_AWS_LAMBDA_INVOKATION_ATTEMPTS = 4 # 1 + 3 retries
+AWS_LAMBDA_RETRY_DELAY_SECONDS = 4 # 4 ^ retry attempt, so retry attempt 3 is delayed 64 seconds
 
 
 class FullRefreshReason: