Fixing issues with incremental csv loading

Author: davidames

appveyor.yml

@@ -47,7 +47,7 @@ build_script:
 test_script:
   - test_full_refresh_from_csv.cmd
   - test_incremental_refresh_from_csv.cmd
-  - test_full_refresh_from_mssql.cmd
+  - test_incremental_refresh_from_mssql.cmd
 
 on_finish:
   #Enable this line to make the build pause after completion for RDP troubleshooting.

integration_tests/mssql_source/source_database_setup/create_database.sql

@@ -1,2 +1,6 @@
 IF NOT EXISTS (SELECT * FROM sys.databases WHERE Name = 'RelationalDataLoaderIntegrationTestSource')
-	CREATE DATABASE	 RelationalDataLoaderIntegrationTestSource
+	CREATE DATABASE	 RelationalDataLoaderIntegrationTestSource
+
+ALTER DATABASE RelationalDataLoaderIntegrationTestSource
+SET CHANGE_TRACKING = ON
+(CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON)

modules/DataLoadManager.py

@@ -50,8 +50,8 @@ def start_single_import(self, target_engine, model_name, requested_full_refresh)
 
         data_load_tracker = DataLoadTracker(model_name, json_data, full_refresh, change_tracking_info, self.correlation_id)
 
-        self.logger.debug(" Change Tracking: this_sync_version: {0} next_sync_version: {1} force_full_load:{2}    : ".format(change_tracking_info.this_sync_version, change_tracking_info.next_sync_version, change_tracking_info.force_full_load()))
-        if not full_refresh and change_tracking_info.force_full_load():
+        self.logger.debug(" Change Tracking: this_sync_version: {0} next_sync_version: {1} force_full_load:{2}    : ".format(change_tracking_info.this_sync_version, change_tracking_info.next_sync_version, change_tracking_info.force_full_load))
+        if not full_refresh and change_tracking_info.force_full_load:
             self.logger.info("Change tracking has forced this to be a full load")
             full_refresh = True
 

modules/data_sources/ChangeTrackingInfo.py

@@ -2,12 +2,9 @@
 class ChangeTrackingInfo:
     this_sync_version = 0
     next_sync_version = 0
-
-    def __init__(self, this_sync_version, next_sync_version):
+    force_full_load = 0
+    def __init__(self, this_sync_version, next_sync_version, force_full_load):
         self.this_sync_version = this_sync_version
         self.next_sync_version = next_sync_version
-
-    def force_full_load(self):
-        return bool(self.this_sync_version == 0 or self.next_sync_version == 0)
-
+        self.force_full_load = force_full_load
 

modules/data_sources/CsvDataSource.py

@@ -64,4 +64,4 @@ def get_next_data_frame(self, table_configuration, columns, batch_configuration,
         return data_frame
 
     def init_change_tracking(self, table_configuration, last_sync_version):
-        return ChangeTrackingInfo(0,0)
+        return ChangeTrackingInfo(0, 0, False)

modules/data_sources/MsSqlDataSource.py

@@ -123,4 +123,5 @@ def init_change_tracking(self, table_configuration, last_sync_version):
         row = result.fetchone()
         sql_builder.close()
 
-        return ChangeTrackingInfo(row["this_sync_version"], row["next_sync_version"])
+        force_full_load = bool(row["this_sync_version"] == 0 or row["next_sync_version"] == 0)
+        return ChangeTrackingInfo(row["this_sync_version"], row["next_sync_version"], force_full_load)