[SP-337] Move method to batch data loader

Author: waltaro

rdl/BatchDataLoader.py

@@ -61,6 +61,10 @@ def load_batch(self, batch_key_tracker):
             batch_key_tracker.has_more_data = False
             return
 
+        # replacing unicode null characters because postgres doesn't support null characters in text fields
+        # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.replace.html
+        data_frame = data_frame.replace(regex=r"\x00", value="")
+
         data_frame = self.attach_column_transformers(data_frame)
         self.write_data_frame_to_table(data_frame)
         batch_tracker.load_completed_successfully()
@@ -122,7 +126,8 @@ def write_data_frame_to_table(self, data_frame):
             f"null '\\N', "
             f"FORCE_NULL ({column_list}))"
         )
-        self.logger.debug(f"Writing to table using command '{sql}'")
+
+        self.logger.info(f"Writing to table using command '{sql}'")
 
         curs.copy_expert(sql=sql, file=data)
 

rdl/data_sources/AWSLambdaDataSource.py

@@ -141,9 +141,7 @@ def __get_table_data(
         return result["ColumnNames"], data
 
     def __get_data_frame(self, data: [[]], column_names: []):
-        return pandas.DataFrame(data=data, columns=column_names).replace(
-            regex=r"\x00", value=""
-        )
+        return pandas.DataFrame(data=data, columns=column_names)
 
     def __invoke_lambda(self, pay_load):
         self.logger.debug("\nRequest being sent to Lambda:")