From c25611d8c130382df992c8cd40f80320b4f269d2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 23:01:10 +0000 Subject: [PATCH 1/6] feat: Support connector_builder_project_id in get_custom_source_definition Allow get_custom_source_definition to accept either definition_id or connector_builder_project_id. Adds validation to ensure exactly one parameter is provided and that connector_builder_project_id is only used with yaml definition_type. - Modified method signature to accept optional definition_id and connector_builder_project_id parameters - Added validation for mutually exclusive parameters - Added validation for yaml-only connector_builder_project_id usage - Implemented lookup by connector_builder_project_id via list and match - Added test coverage for new parameter validation - Updated existing tests to use keyword arguments Requested by: AJ Steers (@aaronsteers) Co-Authored-By: AJ Steers --- airbyte/cloud/workspaces.py | 67 ++++++++++++++++--- .../cloud/test_custom_definitions.py | 44 +++++++++++- 2 files changed, 99 insertions(+), 12 deletions(-) diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index af201e4c..9b6be340 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -583,28 +583,73 @@ def list_custom_source_definitions( def get_custom_source_definition( self, - definition_id: str, *, + definition_id: str | None = None, definition_type: Literal["yaml", "docker"], + connector_builder_project_id: str | None = None, ) -> CustomCloudSourceDefinition: - """Get a specific custom source definition by ID. + """Get a specific custom source definition by ID or builder project ID. Args: - definition_id: The definition ID + definition_id: The definition ID (optional if connector_builder_project_id is provided) definition_type: Connector type ("yaml" or "docker"). Required. + connector_builder_project_id: The connector builder project ID (optional, only valid + for "yaml" definition_type) Returns: CustomCloudSourceDefinition object + + Raises: + PyAirbyteInputError: If both or neither parameters are provided, or if + connector_builder_project_id is used with non-yaml definition_type """ - if definition_type == "yaml": - result = api_util.get_custom_yaml_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, + if (definition_id is None) == (connector_builder_project_id is None): + raise exc.PyAirbyteInputError( + message=( + "Must specify EXACTLY ONE of definition_id or connector_builder_project_id" + ), + context={ + "definition_id": definition_id, + "connector_builder_project_id": connector_builder_project_id, + }, ) - return CustomCloudSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 + + if connector_builder_project_id is not None and definition_type != "yaml": + raise exc.PyAirbyteInputError( + message="connector_builder_project_id is only valid for yaml definition_type", + context={ + "definition_type": definition_type, + "connector_builder_project_id": connector_builder_project_id, + }, + ) + + if definition_type == "yaml": + if definition_id is not None: + result = api_util.get_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return CustomCloudSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 + + if connector_builder_project_id is not None: + definitions = self.list_custom_source_definitions(definition_type="yaml") + for definition in definitions: + if definition.connector_builder_project_id == connector_builder_project_id: + return definition + + raise exc.AirbyteError( + message=( + "No custom source definition found with the given " + "connector_builder_project_id" + ), + context={ + "workspace_id": self.workspace_id, + "connector_builder_project_id": connector_builder_project_id, + }, + ) raise NotImplementedError( "Docker custom source definitions are not yet supported. " diff --git a/tests/integration_tests/cloud/test_custom_definitions.py b/tests/integration_tests/cloud/test_custom_definitions.py index fbabf469..89a79405 100644 --- a/tests/integration_tests/cloud/test_custom_definitions.py +++ b/tests/integration_tests/cloud/test_custom_definitions.py @@ -77,7 +77,7 @@ def test_publish_custom_yaml_source( assert definitions[0].definition_id == definition_id fetched = cloud_workspace.get_custom_source_definition( - definition_id, + definition_id=definition_id, definition_type="yaml", ) assert fetched.definition_id == definition_id @@ -85,6 +85,15 @@ def test_publish_custom_yaml_source( assert fetched.definition_type == "yaml" assert fetched.connector_type == "source" + builder_project_id = fetched.connector_builder_project_id + if builder_project_id: + fetched_by_project_id = cloud_workspace.get_custom_source_definition( + connector_builder_project_id=builder_project_id, + definition_type="yaml", + ) + assert fetched_by_project_id.definition_id == definition_id + assert fetched_by_project_id.name == name + updated_manifest = TEST_YAML_MANIFEST.copy() updated_manifest["version"] = "0.2.0" updated = fetched.update_definition( @@ -168,3 +177,36 @@ def test_safe_mode_deletion( finally: definition.permanently_delete(safe_mode=False) + + +def test_get_custom_source_definition_validation() -> None: + """Test that get_custom_source_definition validates input parameters correctly.""" + from airbyte.exceptions import PyAirbyteInputError + from unittest.mock import Mock + + workspace = Mock(spec=CloudWorkspace) + workspace.workspace_id = "test-workspace" + + with pytest.raises(PyAirbyteInputError) as exc_info: + CloudWorkspace.get_custom_source_definition( + workspace, + definition_type="yaml", + ) + assert "EXACTLY ONE" in str(exc_info.value) + + with pytest.raises(PyAirbyteInputError) as exc_info: + CloudWorkspace.get_custom_source_definition( + workspace, + definition_id="test-id", + definition_type="yaml", + connector_builder_project_id="test-project-id", + ) + assert "EXACTLY ONE" in str(exc_info.value) + + with pytest.raises(PyAirbyteInputError) as exc_info: + CloudWorkspace.get_custom_source_definition( + workspace, + definition_type="docker", + connector_builder_project_id="test-project-id", + ) + assert "only valid for yaml" in str(exc_info.value) From 4bbe0c695eb0219edd5941fffd92530998adb31a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 23:16:22 +0000 Subject: [PATCH 2/6] refactor: Use API endpoint for builder_project_id lookup Replace list-and-filter approach with direct API call to /connector_builder_projects/get_with_manifest endpoint. - Added get_definition_id_for_connector_builder_project() API function - Updated get_custom_source_definition to use the new API function - More efficient reverse lookup from builder_project_id to definition_id Suggested by: AJ Steers (@aaronsteers) Co-Authored-By: AJ Steers --- airbyte/_util/api_util.py | 38 ++++++++++++++++++++++++++++++++++ airbyte/cloud/workspaces.py | 41 ++++++++++++++++++++++++------------- 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/airbyte/_util/api_util.py b/airbyte/_util/api_util.py index 3f72bcfb..0eac83ad 100644 --- a/airbyte/_util/api_util.py +++ b/airbyte/_util/api_util.py @@ -1418,3 +1418,41 @@ def get_connector_builder_project_for_definition_id( client_secret=client_secret, ) return json_result.get("builderProjectId") + + +def get_definition_id_for_connector_builder_project( + *, + workspace_id: str, + builder_project_id: str, + api_root: str, + client_id: SecretString, + client_secret: SecretString, +) -> str | None: + """Get the source definition ID for a connector builder project. + + Uses the Config API endpoint: + /v1/connector_builder_projects/get_with_manifest + + See: https://github.com/airbytehq/airbyte-platform-internal/blob/master/oss/airbyte-api/server-api/src/main/openapi/config.yaml#L11900 + + Args: + workspace_id: The workspace ID + builder_project_id: The connector builder project ID + api_root: The API root URL + client_id: OAuth client ID + client_secret: OAuth client secret + + Returns: + The source definition ID if found, None otherwise (can be null in API response) + """ + json_result = _make_config_api_request( + path="/connector_builder_projects/get_with_manifest", + json={ + "builderProjectId": builder_project_id, + "workspaceId": workspace_id, + }, + api_root=api_root, + client_id=client_id, + client_secret=client_secret, + ) + return json_result.get("sourceDefinitionId") diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 9b6be340..add30e21 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -635,21 +635,34 @@ def get_custom_source_definition( return CustomCloudSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 if connector_builder_project_id is not None: - definitions = self.list_custom_source_definitions(definition_type="yaml") - for definition in definitions: - if definition.connector_builder_project_id == connector_builder_project_id: - return definition - - raise exc.AirbyteError( - message=( - "No custom source definition found with the given " - "connector_builder_project_id" - ), - context={ - "workspace_id": self.workspace_id, - "connector_builder_project_id": connector_builder_project_id, - }, + definition_id = api_util.get_definition_id_for_connector_builder_project( + workspace_id=self.workspace_id, + builder_project_id=connector_builder_project_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + + if definition_id is None: + raise exc.AirbyteError( + message=( + "No custom source definition found with the given " + "connector_builder_project_id" + ), + context={ + "workspace_id": self.workspace_id, + "connector_builder_project_id": connector_builder_project_id, + }, + ) + + result = api_util.get_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, ) + return CustomCloudSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 raise NotImplementedError( "Docker custom source definitions are not yet supported. " From e0af59cd47e40db867eccb6ce5415228e1d0d45a Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 16 Oct 2025 16:28:12 -0700 Subject: [PATCH 3/6] clean up Clarify argument requirements and exclusivity in get_custom_source_definition. --- airbyte/cloud/workspaces.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index add30e21..b0102512 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -583,18 +583,18 @@ def list_custom_source_definitions( def get_custom_source_definition( self, - *, definition_id: str | None = None, + *, definition_type: Literal["yaml", "docker"], connector_builder_project_id: str | None = None, ) -> CustomCloudSourceDefinition: """Get a specific custom source definition by ID or builder project ID. Args: - definition_id: The definition ID (optional if connector_builder_project_id is provided) + definition_id: The definition ID. Mutually exclusive with `connector_builder_project_id`. definition_type: Connector type ("yaml" or "docker"). Required. - connector_builder_project_id: The connector builder project ID (optional, only valid - for "yaml" definition_type) + connector_builder_project_id: The connector builder project ID. Mutually exclusive with + `definition_id`. Returns: CustomCloudSourceDefinition object From 475d109bc7d06e7be22b7e7e1246ad7d3cce87fa Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 23:39:38 +0000 Subject: [PATCH 4/6] fix: Fix lint and raise error when builder_project not found - Fix line length issue in docstring (E501) - Changed get_definition_id_for_connector_builder_project to raise AirbyteError instead of returning None when definition not found - Simplified caller code in get_custom_source_definition to remove None check since API function now raises directly Fixes lint error and addresses feedback from @aaronsteers Co-Authored-By: AJ Steers --- airbyte/_util/api_util.py | 18 +++++++++++++++--- airbyte/cloud/workspaces.py | 19 ++++--------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/airbyte/_util/api_util.py b/airbyte/_util/api_util.py index 0eac83ad..4d2886f4 100644 --- a/airbyte/_util/api_util.py +++ b/airbyte/_util/api_util.py @@ -1427,7 +1427,7 @@ def get_definition_id_for_connector_builder_project( api_root: str, client_id: SecretString, client_secret: SecretString, -) -> str | None: +) -> str: """Get the source definition ID for a connector builder project. Uses the Config API endpoint: @@ -1443,7 +1443,10 @@ def get_definition_id_for_connector_builder_project( client_secret: OAuth client secret Returns: - The source definition ID if found, None otherwise (can be null in API response) + The source definition ID + + Raises: + AirbyteError: If no definition is found for the given builder project ID """ json_result = _make_config_api_request( path="/connector_builder_projects/get_with_manifest", @@ -1455,4 +1458,13 @@ def get_definition_id_for_connector_builder_project( client_id=client_id, client_secret=client_secret, ) - return json_result.get("sourceDefinitionId") + definition_id = json_result.get("sourceDefinitionId") + if definition_id is None: + raise AirbyteError( + message="No source definition found for the given connector builder project", + context={ + "workspace_id": workspace_id, + "builder_project_id": builder_project_id, + }, + ) + return definition_id diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index b0102512..71c41907 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -591,10 +591,11 @@ def get_custom_source_definition( """Get a specific custom source definition by ID or builder project ID. Args: - definition_id: The definition ID. Mutually exclusive with `connector_builder_project_id`. + definition_id: The definition ID. Mutually exclusive with + `connector_builder_project_id`. definition_type: Connector type ("yaml" or "docker"). Required. - connector_builder_project_id: The connector builder project ID. Mutually exclusive with - `definition_id`. + connector_builder_project_id: The connector builder project ID. + Mutually exclusive with `definition_id` Returns: CustomCloudSourceDefinition object @@ -643,18 +644,6 @@ def get_custom_source_definition( client_secret=self.client_secret, ) - if definition_id is None: - raise exc.AirbyteError( - message=( - "No custom source definition found with the given " - "connector_builder_project_id" - ), - context={ - "workspace_id": self.workspace_id, - "connector_builder_project_id": connector_builder_project_id, - }, - ) - result = api_util.get_custom_yaml_source_definition( workspace_id=self.workspace_id, definition_id=definition_id, From 2e9ec5bf4c578312bb2b66727c3b575e52208539 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 16 Oct 2025 17:13:03 -0700 Subject: [PATCH 5/6] simplify --- airbyte/cloud/workspaces.py | 93 ++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 71c41907..3569a213 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -604,6 +604,12 @@ def get_custom_source_definition( PyAirbyteInputError: If both or neither parameters are provided, or if connector_builder_project_id is used with non-yaml definition_type """ + if definition_type != "yaml": + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." + ) + if (definition_id is None) == (connector_builder_project_id is None): raise exc.PyAirbyteInputError( message=( @@ -615,45 +621,56 @@ def get_custom_source_definition( }, ) - if connector_builder_project_id is not None and definition_type != "yaml": - raise exc.PyAirbyteInputError( - message="connector_builder_project_id is only valid for yaml definition_type", - context={ - "definition_type": definition_type, - "connector_builder_project_id": connector_builder_project_id, - }, + if connector_builder_project_id: + if definition_type != "yaml": + raise exc.PyAirbyteInputError( + message="connector_builder_project_id is only valid for yaml definition_type", + context={ + "definition_type": definition_type, + "connector_builder_project_id": connector_builder_project_id, + }, + ) + definition_id = api_util.get_definition_id_for_connector_builder_project( + workspace_id=self.workspace_id, + builder_project_id=connector_builder_project_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, ) - if definition_type == "yaml": - if definition_id is not None: - result = api_util.get_custom_yaml_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return CustomCloudSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 - - if connector_builder_project_id is not None: - definition_id = api_util.get_definition_id_for_connector_builder_project( - workspace_id=self.workspace_id, - builder_project_id=connector_builder_project_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) + # Definition ID is guaranteed to be set by here + result = api_util.get_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + return CustomCloudSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 - result = api_util.get_custom_yaml_source_definition( - workspace_id=self.workspace_id, - definition_id=definition_id, - api_root=self.api_root, - client_id=self.client_id, - client_secret=self.client_secret, - ) - return CustomCloudSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 - raise NotImplementedError( - "Docker custom source definitions are not yet supported. " - "Only YAML manifest-based custom sources are currently available." - ) + def permanently_delete_custom_source_definition( + self, + definition_id: str, + *, + definition_type: Literal["yaml", "docker"], + ) -> None: + """Permanently delete a custom source definition. + + Args: + definition_id: The definition ID to delete + definition_type: Connector type ("yaml" or "docker"). Required. + """ + if definition_type == "yaml": + api_util.delete_custom_yaml_source_definition( + workspace_id=self.workspace_id, + definition_id=definition_id, + api_root=self.api_root, + client_id=self.client_id, + client_secret=self.client_secret, + ) + else: + raise NotImplementedError( + "Docker custom source definitions are not yet supported. " + "Only YAML manifest-based custom sources are currently available." + ) From 44309a889fdc7dabfe6a229e74625fdfbbaf2ce5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 17 Oct 2025 00:16:19 +0000 Subject: [PATCH 6/6] fix: Add type assertion for definition_id Mypy wasn't able to infer that definition_id is guaranteed to be set, so added an assertion to help the type checker. Co-Authored-By: AJ Steers --- airbyte/cloud/workspaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte/cloud/workspaces.py b/airbyte/cloud/workspaces.py index 3569a213..ad58cdad 100644 --- a/airbyte/cloud/workspaces.py +++ b/airbyte/cloud/workspaces.py @@ -639,6 +639,7 @@ def get_custom_source_definition( ) # Definition ID is guaranteed to be set by here + assert definition_id is not None result = api_util.get_custom_yaml_source_definition( workspace_id=self.workspace_id, definition_id=definition_id, @@ -648,7 +649,6 @@ def get_custom_source_definition( ) return CustomCloudSourceDefinition._from_yaml_response(self, result) # noqa: SLF001 - def permanently_delete_custom_source_definition( self, definition_id: str,