minor fixes in pipedrive_pipeline,py

Author: dat-a-man

sources/pipedrive_pipeline.py

@@ -10,10 +10,9 @@ def load_pipedrive() -> None:
     # configure the pipeline with your destination details
     pipeline = dlt.pipeline(
         pipeline_name="pipedrive",
-        destination="duckdb",
+        destination='duckdb',
         dataset_name="pipedrive_data",
         progress="log",
-        dev_mode=True,
     )
     load_info = pipeline.run(pipedrive_source())
     print(load_info)
@@ -24,17 +23,15 @@ def load_selected_data() -> None:
     """Shows how to load just selected tables using `with_resources`"""
     pipeline = dlt.pipeline(
         pipeline_name="pipedrive",
-        destination="duckdb",
+        destination='duckdb',
         dataset_name="pipedrive_data",
         progress="log",
-        dev_mode=True,
     )
     # Use with_resources to select which entities to load
     # Note: `custom_fields_mapping` must be included to translate custom field hashes to corresponding names
     load_info = pipeline.run(
         pipedrive_source().with_resources(
-            # "products", "deals", "deals_participants", "custom_fields_mapping"
-            "deals"
+             "products", "deals", "deals_participants", "custom_fields_mapping"
         )
     )
     print(load_info)
@@ -57,10 +54,9 @@ def load_from_start_date() -> None:
     """Example to incrementally load activities limited to items updated after a given date"""
     pipeline = dlt.pipeline(
         pipeline_name="pipedrive",
-        destination="duckdb",
+        destination='duckdb',
         dataset_name="pipedrive_data",
         progress="log",
-        dev_mode=True,
     )
 
     # First source configure to load everything except activities from the beginning
@@ -78,35 +74,36 @@ def load_from_start_date() -> None:
 
 
 def load_v2_resources(resources: Optional[Sequence[str]] = None) -> None:
-    """Load v2 entities using the separate v2 source.
+    """Load v2 entities using the separate v2 source."""
 
-    Note: company_domain will be read from dlt secrets if not provided.
-    """
     resources = list(resources or DEFAULT_V2_RESOURCES)
     pipeline = dlt.pipeline(
         pipeline_name="pipedrive",
-        destination="duckdb",
+        destination='duckdb',
         dataset_name="pipedrive_data",
         progress="log",
-        dev_mode=True,
     )
     source = pipedrive_v2_source(resources=resources)
     load_info = pipeline.run(source)
     print(load_info)
     print(pipeline.last_trace.last_normalize_info)
 
 
-def load_selected_v2_data(resources: Sequence[str]) -> None:
+def load_selected_v2_data() -> None:
     """Load only the specified v2 entities (and their nested resources)."""
     pipeline = dlt.pipeline(
-        pipeline_name="pipedrive",
+        pipeline_name="pipedrive21",
         destination="duckdb",
-        dataset_name="pipedrive_data",
+        dataset_name="pipedrive_dat21a",
         progress="log",
-        dev_mode=True,
     )
-    source = pipedrive_v2_source(resources=list(resources))
-    load_info = pipeline.run(source)
+    # Use with_resources to select which entities to load (3 major v2 endpoints)
+    # Note: v2 resources are prefixed with "v2_" in the source
+    load_info = pipeline.run(
+        pipedrive_v2_source().with_resources(
+            "v2_deals", "v2_persons", "v2_organizations"
+        )
+    )
     print(load_info)
     print(pipeline.last_trace.last_normalize_info)
 
@@ -119,6 +116,6 @@ def load_selected_v2_data(resources: Sequence[str]) -> None:
     # load activities updated since given date
     # load_from_start_date()
     # load v2 resources (optional addon)
-    load_v2_resources()
-    # load only selected v2 resources
-    # load_selected_v2_data(["deals", "stages"])
+    #load_v2_resources()
+    # load only selected v2 resources (3 major endpoints: deals, persons, organizations)
+    load_selected_v2_data()