diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 23ad0571..dada3f75 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -12,8 +12,19 @@ jobs: python-autofix: runs-on: ubuntu-latest steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: diff --git a/.github/workflows/fix-pr-command.yml b/.github/workflows/fix-pr-command.yml index a2b4cbed..e4decda9 100644 --- a/.github/workflows/fix-pr-command.yml +++ b/.github/workflows/fix-pr-command.yml @@ -40,7 +40,7 @@ jobs: id: get-app-token with: owner: "airbytehq" - repositories: "PyAirbyte" + repositories: "PyAirbyte,sonar" app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout Airbyte @@ -49,6 +49,7 @@ jobs: # Important that this is set so that CI checks are triggered again # Without this we would be forever waiting on required checks to pass token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Checkout PR (${{ github.event.inputs.pr }}) uses: dawidd6/action-checkout-pr@a7598e18433a763b784f17d666372913d8bd4205 # v1.2.0 diff --git a/.github/workflows/pydoc_preview.yml b/.github/workflows/pydoc_preview.yml index 53fb3d59..feaeb500 100644 --- a/.github/workflows/pydoc_preview.yml +++ b/.github/workflows/pydoc_preview.yml @@ -14,8 +14,19 @@ jobs: runs-on: ubuntu-latest steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: diff --git a/.github/workflows/pydoc_publish.yml b/.github/workflows/pydoc_publish.yml index 7c8f6dfd..c44045aa 100644 --- a/.github/workflows/pydoc_publish.yml +++ b/.github/workflows/pydoc_publish.yml @@ -31,8 +31,19 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index d89c1e5a..a9fe6a28 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -17,8 +17,19 @@ jobs: runs-on: ubuntu-latest steps: # Common steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Poetry uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9 with: @@ -43,8 +54,19 @@ jobs: runs-on: ubuntu-latest steps: # Common steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: @@ -65,8 +87,19 @@ jobs: runs-on: ubuntu-latest steps: # Common steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Poetry uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9 with: diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index d26df6b4..853a0ab8 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -25,8 +25,19 @@ jobs: runs-on: ubuntu-latest steps: # Common steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Poetry uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9 with: @@ -90,8 +101,19 @@ jobs: runs-on: ubuntu-latest steps: # Common steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Poetry uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9 with: @@ -168,8 +190,19 @@ jobs: PYTHONIOENCODING: utf-8 steps: # Common steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: @@ -241,8 +274,19 @@ jobs: name: Dependency Analysis with Deptry runs-on: ubuntu-latest steps: + - name: Authenticate as GitHub App + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + id: get-app-token + with: + owner: "airbytehq" + repositories: "PyAirbyte,sonar" + app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} + private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} - name: Checkout code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + token: ${{ steps.get-app-token.outputs.token }} + submodules: recursive - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: @@ -256,5 +300,4 @@ jobs: # Job-specific step(s): - name: Run Deptry - run: | - poetry run deptry . + run: poetry run poe check-deps diff --git a/.github/workflows/test-pr-command.yml b/.github/workflows/test-pr-command.yml index 48ffa86e..b7ab710d 100644 --- a/.github/workflows/test-pr-command.yml +++ b/.github/workflows/test-pr-command.yml @@ -81,7 +81,7 @@ jobs: id: get-app-token with: owner: "airbytehq" - repositories: "PyAirbyte" + repositories: "PyAirbyte,sonar" app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }} private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }} @@ -90,6 +90,7 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ needs.start-workflow.outputs.commit-sha }} + submodules: recursive # Post "In Progress" status to the PR. # This is required because otherwise slash commands won't automatically diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..f01bbe67 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "sonar"] + path = sonar + url = https://github.com/airbytehq/sonar.git diff --git a/.ruff.toml b/.ruff.toml index f0b61926..e9db59ab 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -3,7 +3,11 @@ target-version = "py310" preview = true line-length = 100 +exclude = [ + "sonar", # Git submodule, not part of PyAirbyte codebase +] +[lint] select = [ # For rules reference, see https://docs.astral.sh/ruff/rules/ "A", # flake8-builtins @@ -61,8 +65,6 @@ select = [ "W", # pycodestyle (warnings) "YTT", # flake8-2020 ] - -[lint] ignore = [ # For rules reference, see https://docs.astral.sh/ruff/rules/ diff --git a/airbyte/__init__.py b/airbyte/__init__.py index 976c5a5f..7cc40e30 100644 --- a/airbyte/__init__.py +++ b/airbyte/__init__.py @@ -132,6 +132,8 @@ from airbyte.datasets import CachedDataset from airbyte.destinations.base import Destination from airbyte.destinations.util import get_destination +from airbyte.integrations.base import Integration +from airbyte.integrations.util import get_integration from airbyte.records import StreamRecord from airbyte.registry import get_available_connectors from airbyte.results import ReadResult, WriteResult @@ -154,6 +156,7 @@ documents, exceptions, # noqa: ICN001 # No 'exc' alias for top-level module experimental, + integrations, logs, mcp, records, @@ -175,6 +178,7 @@ "documents", "exceptions", "experimental", + "integrations", "logs", "mcp", "records", @@ -187,6 +191,7 @@ "get_colab_cache", "get_default_cache", "get_destination", + "get_integration", "get_secret", "get_source", "new_local_cache", @@ -195,6 +200,7 @@ "CachedDataset", "Destination", "DuckDBCache", + "Integration", "ReadResult", "SecretSourceEnum", "Source", diff --git a/airbyte/_connector_base.py b/airbyte/_connector_base.py index 0e8a66a4..608d9194 100644 --- a/airbyte/_connector_base.py +++ b/airbyte/_connector_base.py @@ -54,7 +54,7 @@ class ConnectorBase(abc.ABC): """A class representing a destination that can be called.""" - connector_type: Literal["destination", "source"] + connector_type: Literal["destination", "source", "integration"] def __init__( self, diff --git a/airbyte/_executors/sonar.py b/airbyte/_executors/sonar.py new file mode 100644 index 00000000..0673c372 --- /dev/null +++ b/airbyte/_executors/sonar.py @@ -0,0 +1,200 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Sonar Integration Executor for YAML-based connectors.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from airbyte import exceptions as exc +from airbyte._executors.base import Executor + + +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Any + + from airbyte.registry import ConnectorMetadata + + +class SonarExecutor(Executor): + """Executor for Sonar YAML-based integration connectors. + + This executor wraps the connector-sdk's ConnectorExecutor to run + YAML-defined connectors without subprocess execution. + """ + + def __init__( + self, + *, + name: str, + yaml_path: Path | str, + secrets: dict[str, Any] | None = None, + enable_logging: bool = False, + log_file: str | None = None, + metadata: ConnectorMetadata | None = None, + target_version: str | None = None, + ) -> None: + """Initialize Sonar executor. + + Args: + name: Connector name + yaml_path: Path to connector YAML definition + secrets: Secret credentials (will be converted to SecretStr) + enable_logging: Enable request/response logging + log_file: Path to log file + metadata: Connector metadata (optional) + target_version: Target version (not used for YAML connectors) + """ + super().__init__(name=name, metadata=metadata, target_version=target_version) + + self.yaml_path = Path(yaml_path) + self.secrets = secrets or {} + self.enable_logging = enable_logging + self.log_file = log_file + self._executor: Any = None # connector_sdk.ConnectorExecutor + + if not self.yaml_path.exists(): + raise exc.PyAirbyteInputError( + message=f"Connector YAML file not found: {yaml_path}", + input_value=str(yaml_path), + ) + + def _get_executor(self) -> Any: # noqa: ANN401 + """Lazily create connector-sdk executor. + + Returns: + ConnectorExecutor instance + + Raises: + PyAirbyteInputError: If connector-sdk is not installed + """ + if self._executor is None: + try: + from connector_sdk import ConnectorExecutor # noqa: PLC0415 + from connector_sdk.secrets import SecretStr # noqa: PLC0415 + except ImportError as ex: + raise exc.PyAirbyteInputError( + message=( + "connector-sdk is required for Sonar integrations. " + "Install it with: poetry install --with integrations" + ), + guidance="Run: poetry install --with integrations", + ) from ex + + # Convert secrets to SecretStr + secret_dict = { + key: SecretStr(value) if not isinstance(value, SecretStr) else value + for key, value in self.secrets.items() + } + + self._executor = ConnectorExecutor( + config_path=str(self.yaml_path), + secrets=secret_dict, + enable_logging=self.enable_logging, + log_file=self.log_file, + ) + + return self._executor + + @property + def _cli(self) -> list[str]: + """Get CLI args (not used for Sonar executor). + + This property is required by the Executor base class but is not + used for YAML-based connectors. + """ + return [] + + def execute( + self, + args: list[str], + *, + stdin: Any = None, # noqa: ANN401 + suppress_stderr: bool = False, + ) -> Iterator[str]: + """Execute is not supported for Sonar connectors. + + Sonar connectors use async execute methods instead of subprocess execution. + Use Integration.execute() or Integration.aexecute() instead. + + Raises: + NotImplementedError: Always raised + """ + raise NotImplementedError( + "Sonar connectors do not support subprocess execution. " + "Use Integration.execute() or Integration.aexecute() instead." + ) + + async def aexecute( + self, + resource: str, + verb: str, + params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Execute a verb on a resource asynchronously. + + Args: + resource: Resource name (e.g., "customers") + verb: Verb to execute (e.g., "list", "get", "create") + params: Parameters for the operation + + Returns: + API response as dictionary + """ + executor = self._get_executor() + return await executor.execute(resource, verb, params) + + async def aexecute_batch( + self, + operations: list[tuple[str, str, dict[str, Any] | None]], + ) -> list[dict[str, Any]]: + """Execute multiple operations concurrently. + + Args: + operations: List of (resource, verb, params) tuples + + Returns: + List of responses in the same order as operations + """ + executor = self._get_executor() + return await executor.execute_batch(operations) + + def ensure_installation(self, *, auto_fix: bool = True) -> None: + """Ensure connector is available (no-op for YAML connectors).""" + pass + + def install(self) -> None: + """Install connector (no-op for YAML connectors).""" + pass + + def uninstall(self) -> None: + """Uninstall connector (no-op for YAML connectors).""" + pass + + def get_installed_version( + self, + *, + raise_on_error: bool = False, + recheck: bool = False, # noqa: ARG002 + ) -> str | None: + """Get connector version from YAML metadata. + + Returns: + Version string if available in YAML, None otherwise + """ + try: + from connector_sdk.config_loader import load_connector_config # noqa: PLC0415 + + config = load_connector_config(str(self.yaml_path)) + except Exception: + if raise_on_error: + raise + return None + else: + return config.connector.version if config.connector else None + + +__all__ = [ + "SonarExecutor", +] diff --git a/airbyte/integrations/__init__.py b/airbyte/integrations/__init__.py new file mode 100644 index 00000000..83b6e549 --- /dev/null +++ b/airbyte/integrations/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Sonar Integration support for PyAirbyte.""" + +from airbyte.integrations.base import Integration +from airbyte.integrations.util import get_integration + + +__all__ = [ + "Integration", + "get_integration", +] diff --git a/airbyte/integrations/base.py b/airbyte/integrations/base.py new file mode 100644 index 00000000..39c9c7a1 --- /dev/null +++ b/airbyte/integrations/base.py @@ -0,0 +1,293 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Integration base class for Sonar connectors.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import TYPE_CHECKING, Any, NoReturn + +from airbyte import exceptions as exc +from airbyte._connector_base import ConnectorBase + + +if TYPE_CHECKING: + from airbyte._executors.sonar import SonarExecutor + + +class Integration(ConnectorBase): + """A class representing a Sonar integration connector. + + Integrations are YAML-based connectors that provide API access through + resource and verb operations, rather than the stream-based model used + by sources and destinations. + + Example: + ```python + from airbyte import get_integration + + integration = get_integration( + name="my-api", yaml_path="./connectors/my-api.yaml", secrets={"api_key": "sk_test_..."} + ) + + result = integration.execute("customers", "list", params={"limit": 10}) + + result = await integration.aexecute("customers", "get", params={"id": "123"}) + ``` + """ + + connector_type = "integration" + + def __init__( + self, + executor: SonarExecutor, + name: str, + yaml_path: Path | str, + *, + validate: bool = False, + ) -> None: + """Initialize the integration. + + Args: + executor: SonarExecutor instance + name: Integration name + yaml_path: Path to connector YAML definition + validate: Whether to validate the YAML on initialization + """ + super().__init__( + executor=executor, + name=name, + config=None, # Integrations don't use config in the same way + validate=False, # Skip base class validation + ) + self.yaml_path = Path(yaml_path) + self._resources: list[str] | None = None + + if validate: + self._validate_yaml() + + def _validate_yaml(self) -> None: + """Validate the connector YAML definition. + + Raises: + PyAirbyteInputError: If YAML is invalid + """ + try: + from connector_sdk.config_loader import load_connector_config # noqa: PLC0415 + + load_connector_config(str(self.yaml_path)) + except ImportError as ex: + raise exc.PyAirbyteInputError( + message=( + "connector-sdk is required for Sonar integrations. " + "Install it with: poetry install --with integrations" + ), + guidance="Run: poetry install --with integrations", + ) from ex + except Exception as ex: + raise exc.PyAirbyteInputError( + message=f"Invalid connector YAML: {ex}", + context={"yaml_path": str(self.yaml_path)}, + ) from ex + + def list_resources(self) -> list[str]: + """List available resources in the connector. + + Returns: + List of resource names + """ + if self._resources is None: + try: + from connector_sdk.config_loader import load_connector_config # noqa: PLC0415 + + config = load_connector_config(str(self.yaml_path)) + self._resources = [r.name for r in config.resources] + except Exception as ex: + raise exc.PyAirbyteInputError( + message=f"Failed to load resources from YAML: {ex}", + context={"yaml_path": str(self.yaml_path)}, + ) from ex + + return self._resources + + def list_verbs(self, resource: str) -> list[str]: + """List available verbs for a resource. + + Args: + resource: Resource name + + Returns: + List of verb names (e.g., ["get", "list", "create"]) + """ + + def _raise_not_found(available: list[str]) -> NoReturn: + raise exc.PyAirbyteInputError( + message=f"Resource '{resource}' not found", + context={ + "resource": resource, + "available_resources": available, + }, + ) + + try: + from connector_sdk.config_loader import load_connector_config # noqa: PLC0415 + + config = load_connector_config(str(self.yaml_path)) + for r in config.resources: + if r.name == resource: + return [v.value for v in r.verbs] + + _raise_not_found([r.name for r in config.resources]) + except Exception as ex: + if isinstance(ex, exc.PyAirbyteInputError): + raise + raise exc.PyAirbyteInputError( + message=f"Failed to load verbs for resource '{resource}': {ex}", + context={"yaml_path": str(self.yaml_path), "resource": resource}, + ) from ex + + async def aexecute( + self, + resource: str, + verb: str, + params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Execute a verb on a resource asynchronously. + + Args: + resource: Resource name (e.g., "customers") + verb: Verb to execute (e.g., "list", "get", "create") + params: Parameters for the operation + + Returns: + API response as dictionary + + Example: + ```python + result = await integration.aexecute("customers", "list", params={"limit": 10}) + ``` + """ + from airbyte._executors.sonar import SonarExecutor # noqa: PLC0415 + + if not isinstance(self.executor, SonarExecutor): + raise exc.PyAirbyteInternalError(message="Executor must be a SonarExecutor instance") + + return await self.executor.aexecute(resource, verb, params) + + def execute( + self, + resource: str, + verb: str, + params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Execute a verb on a resource synchronously. + + This is a convenience wrapper around aexecute() that handles + the event loop automatically. + + Args: + resource: Resource name (e.g., "customers") + verb: Verb to execute (e.g., "list", "get", "create") + params: Parameters for the operation + + Returns: + API response as dictionary + + Example: + ```python + result = integration.execute("customers", "get", params={"id": "cus_123"}) + ``` + """ + try: + asyncio.get_running_loop() + raise exc.PyAirbyteInputError( + message=( + "Cannot call execute() from within an async context. " "Use aexecute() instead." + ), + guidance="Use await integration.aexecute(...) in async code", + ) + except RuntimeError: + return asyncio.run(self.aexecute(resource, verb, params)) + + async def aexecute_batch( + self, + operations: list[tuple[str, str, dict[str, Any] | None]], + ) -> list[dict[str, Any]]: + """Execute multiple operations concurrently. + + Args: + operations: List of (resource, verb, params) tuples + + Returns: + List of responses in the same order as operations + + Example: + ```python + results = await integration.aexecute_batch( + [ + ("customers", "list", {"limit": 10}), + ("customers", "get", {"id": "cus_123"}), + ("products", "list", {"limit": 5}), + ] + ) + ``` + """ + from airbyte._executors.sonar import SonarExecutor # noqa: PLC0415 + + if not isinstance(self.executor, SonarExecutor): + raise exc.PyAirbyteInternalError(message="Executor must be a SonarExecutor instance") + + return await self.executor.aexecute_batch(operations) + + def execute_batch( + self, + operations: list[tuple[str, str, dict[str, Any] | None]], + ) -> list[dict[str, Any]]: + """Execute multiple operations concurrently (sync wrapper). + + Args: + operations: List of (resource, verb, params) tuples + + Returns: + List of responses in the same order as operations + """ + try: + asyncio.get_running_loop() + raise exc.PyAirbyteInputError( + message=( + "Cannot call execute_batch() from within an async context. " + "Use aexecute_batch() instead." + ), + guidance="Use await integration.aexecute_batch(...) in async code", + ) + except RuntimeError: + return asyncio.run(self.aexecute_batch(operations)) + + def check(self) -> None: + """Check connector connectivity. + + For Sonar integrations, this is a no-op unless the YAML defines + an 'authorize' verb that can be used for health checks. + + Raises: + NotImplementedError: If no health check mechanism is available + """ + try: + resources = self.list_resources() + for resource in resources: + verbs = self.list_verbs(resource) + if "authorize" in verbs: + self.execute(resource, "authorize", params={}) + return + except Exception: + pass + + raise NotImplementedError( + "This integration does not define a health check mechanism. " + "Add an 'authorize' verb to enable connectivity checks." + ) + + +__all__ = [ + "Integration", +] diff --git a/airbyte/integrations/util.py b/airbyte/integrations/util.py new file mode 100644 index 00000000..f6649ba5 --- /dev/null +++ b/airbyte/integrations/util.py @@ -0,0 +1,69 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Utility functions for working with integrations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from airbyte._executors.sonar import SonarExecutor +from airbyte.integrations.base import Integration + + +if TYPE_CHECKING: + from pathlib import Path + + +def get_integration( + name: str, + yaml_path: Path | str, + secrets: dict[str, Any] | None = None, + *, + enable_logging: bool = False, + log_file: str | None = None, + validate: bool = False, +) -> Integration: + """Get a Sonar integration connector. + + Args: + name: Integration name + yaml_path: Path to connector YAML definition + secrets: Secret credentials for authentication + enable_logging: Enable request/response logging + log_file: Path to log file (if enable_logging=True) + validate: Whether to validate the YAML on initialization + + Returns: + Integration instance + + Example: + ```python + from airbyte import get_integration + + integration = get_integration( + name="my-api", yaml_path="./connectors/my-api.yaml", secrets={"api_key": "sk_test_..."} + ) + + resources = integration.list_resources() + + result = integration.execute("customers", "list", params={"limit": 10}) + ``` + """ + executor = SonarExecutor( + name=name, + yaml_path=yaml_path, + secrets=secrets, + enable_logging=enable_logging, + log_file=log_file, + ) + + return Integration( + executor=executor, + name=name, + yaml_path=yaml_path, + validate=validate, + ) + + +__all__ = [ + "get_integration", +] diff --git a/poetry.lock b/poetry.lock index bcb9c288..6bf4d3f0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -131,7 +131,7 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, @@ -184,7 +184,7 @@ version = "4.10.0" description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1"}, @@ -285,7 +285,7 @@ version = "2.2.1" description = "Function decoration for backoff and retry" optional = false python-versions = ">=3.7,<4.0" -groups = ["main"] +groups = ["main", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, @@ -407,7 +407,7 @@ version = "2025.8.3" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5"}, @@ -501,7 +501,7 @@ version = "3.4.3" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72"}, @@ -591,7 +591,7 @@ version = "8.2.1" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"}, @@ -631,12 +631,40 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "(python_version <= \"3.11\" or python_version >= \"3.12\") and platform_system == \"Windows\"", dev = "python_version <= \"3.11\" or python_version >= \"3.12\""} +markers = {main = "(python_version <= \"3.11\" or python_version >= \"3.12\") and platform_system == \"Windows\"", dev = "python_version <= \"3.11\" or python_version >= \"3.12\"", integrations = "(python_version <= \"3.11\" or python_version >= \"3.12\") and platform_system == \"Windows\""} + +[[package]] +name = "connector-sdk" +version = "0.1.0" +description = "Core SDK for Airbyte connectors with declarative YAML execution" +optional = false +python-versions = ">=3.9" +groups = ["integrations"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [] +develop = true + +[package.dependencies] +click = ">=8.0.0" +httpx = ">=0.24.0" +jinja2 = ">=3.0.0" +jsonref = ">=1.1.0" +pydantic = ">=2.0.0" +python-dotenv = ">=1.0.0" +pyyaml = ">=6.0" +segment-analytics-python = ">=2.2.0" + +[package.extras] +dev = ["pytest (>=7.0.0)", "pytest-asyncio (>=0.21.0)", "pytest-httpx (>=0.30.0)", "pytest-mock (>=3.10.0)", "ruff (==0.7.3)"] + +[package.source] +type = "directory" +url = "sonar/connector-sdk" [[package]] name = "coverage" @@ -1124,12 +1152,12 @@ version = "1.3.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, ] -markers = {main = "python_version <= \"3.11\" or python_version >= \"3.12\"", dev = "python_version < \"3.11\""} +markers = {main = "python_version <= \"3.11\" or python_version >= \"3.12\"", dev = "python_version < \"3.11\"", integrations = "python_version < \"3.11\""} [package.dependencies] typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} @@ -1844,7 +1872,7 @@ version = "0.16.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, @@ -1879,7 +1907,7 @@ version = "1.0.9" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, @@ -1902,7 +1930,7 @@ version = "0.28.1" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, @@ -1981,7 +2009,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, @@ -2064,7 +2092,7 @@ version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, @@ -2239,7 +2267,7 @@ version = "1.1.0" description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["main", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "jsonref-1.1.0-py3-none-any.whl", hash = "sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9"}, @@ -2472,7 +2500,7 @@ version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, @@ -3603,7 +3631,7 @@ version = "2.11.7" description = "Data validation using Python type hints" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b"}, @@ -3702,7 +3730,7 @@ version = "2.33.2" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "pydantic_core-2.33.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8"}, @@ -3900,7 +3928,7 @@ version = "2.10.1" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"}, @@ -4008,6 +4036,26 @@ tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "0.23.8" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "pytest_asyncio-0.23.8-py3-none-any.whl", hash = "sha256:50265d892689a5faefb84df80819d1ecef566eb3549cf915dfb33569359d1ce2"}, + {file = "pytest_asyncio-0.23.8.tar.gz", hash = "sha256:759b10b33a6dc61cce40a8bd5205e302978bbbcc00e279a8b61d9a6a3c82e4d3"}, +] + +[package.dependencies] +pytest = ">=7.0.0,<9" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] + [[package]] name = "pytest-docker" version = "3.2.3" @@ -4070,7 +4118,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, @@ -4086,7 +4134,7 @@ version = "1.1.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "python_dotenv-1.1.1-py3-none-any.whl", hash = "sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc"}, @@ -4175,7 +4223,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, @@ -4461,7 +4509,7 @@ version = "2.32.5" description = "Python HTTP for Humans." optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, @@ -4869,6 +4917,28 @@ botocore = ">=1.37.4,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] +[[package]] +name = "segment-analytics-python" +version = "2.3.4" +description = "The hassle-free way to integrate analytics into any python application." +optional = false +python-versions = ">=3.9.0" +groups = ["integrations"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "segment_analytics_python-2.3.4-py2.py3-none-any.whl", hash = "sha256:ea2b6432bef1c017d24f69bf5f778e45c2385ec9df527242e2b42ee6cefd7367"}, + {file = "segment_analytics_python-2.3.4.tar.gz", hash = "sha256:f188e864cc0fed9fb141aff73c21bfaab783a1dd1b97ab1eff44340aed68f50b"}, +] + +[package.dependencies] +backoff = ">=2.1,<3.0" +PyJWT = ">=2.10.1,<2.11.0" +python-dateutil = ">=2.2,<3.0" +requests = ">=2.7,<3.0" + +[package.extras] +test = ["flake8 (==3.7.9)", "mock (==2.0.0)", "pylint (==3.3.1)"] + [[package]] name = "serpyco-rs" version = "1.16.0" @@ -4953,7 +5023,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, @@ -4966,7 +5036,7 @@ version = "1.3.1" description = "Sniff out which async library your code is running under" optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, @@ -5477,7 +5547,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, @@ -5507,7 +5577,7 @@ version = "0.4.1" description = "Runtime typing introspection tools" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51"}, @@ -5587,7 +5657,7 @@ version = "2.5.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "integrations"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, @@ -5988,4 +6058,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "e5721e804e402c96e3640cc88cf97683f26b0c9ab26fb29bafeb863344fa41bd" +content-hash = "5eb38fdc64601ce178b8a1a7562e98ddc7179a195fb30774007e30ffec02d6ae" diff --git a/pyproject.toml b/pyproject.toml index 0117343b..95f0c050 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,9 @@ uuid7 = "^0.1.0" fastmcp = ">=2.11.3,<3.0.0" uv = ">=0.5.0,<0.9.0" +[tool.poetry.group.integrations.dependencies] +connector-sdk = {path = "./sonar/connector-sdk", develop = true} + [tool.poetry.group.dev.dependencies] coverage = "^7.5.1" deptry = "^0.21.1" @@ -66,6 +69,7 @@ pandas-stubs = "^2.1.4.231218" pdoc = "^16.0.0" poethepoet = ">=0.26.1,<0.32.0" pytest = "^8.2.0" +pytest-asyncio = "^0.23.0" pytest-docker = "^3.1.1" pytest-mock = "^3.14.0" pytest-timeout = "^2.3.1" @@ -88,13 +92,14 @@ build-backend = "poetry_dynamic_versioning.backend" # - No test can take longer than 10 minutes (600 seconds) # - Markers must be declared explicitly # - Generate junit test results at a deterministic location -addopts = "-rs --strict-markers --timeout=600 --junit-xml=build/test-results/test-results.xml" +addopts = "-rs --strict-markers --timeout=600 --junit-xml=build/test-results/test-results.xml --ignore=sonar" markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", "super_slow: these super slow tests will not run in CI; they will only ever run on-demand", "requires_creds: marks a test as requiring credentials (skip when secrets unavailable)", "linting: marks a test as a linting test", "flaky: marks a test as flaky", + "asyncio: marks tests as async (automatically applied by pytest-asyncio)", ] filterwarnings = [ # syntax: "action:message_regex:category:module:line" # Treat python warnings as errors in pytest @@ -144,6 +149,7 @@ coverage-html = { shell = "coverage html -d htmlcov && open htmlcov/index.html" coverage-reset = { shell = "coverage erase" } check = { shell = "ruff check . && pyrefly check && pytest --collect-only -qq" } +check-deps = { shell = "deptry airbyte" } docs-generate = {env = {PDOC_ALLOW_EXEC = "1"}, cmd = "python -m docs.generate run"} docs-preview = {shell = "poe docs-generate && open docs/generated/index.html"} @@ -165,13 +171,21 @@ poe_tasks = ["test"] required_environment_variables = ["GCP_GSM_CREDENTIALS"] side_car_docker_engine = true +[tool.deptry] +exclude = [".venv", "sonar", "tests"] + [tool.deptry.per_rule_ignores] # This is a mapping of rules and package names to be ignored for that rule. DEP001 = [ "IPython" # Optional dependency, used for detecting Notebook environments ] +DEP003 = [ + "airbyte" # Ignore self-imports (airbyte package importing from airbyte submodules) +] DEP004 = [ - "pdoc" # Only used for generating documentation. Not a runtime dependency. + "pdoc", # Only used for generating documentation. Not a runtime dependency. + "connector-sdk", # Optional dependency in integrations group, imported lazily + "connector_sdk", # Module name for connector-sdk (underscore vs hyphen) ] DEP002 = [ # Only used for SQLAlchemy engine. Not imported directly: @@ -181,4 +195,6 @@ DEP002 = [ "sqlalchemy-bigquery", # Used as subprocess tool, not imported directly: "uv", + # Optional dependency in integrations group, imported lazily: + "connector-sdk", ] diff --git a/sonar b/sonar new file mode 160000 index 00000000..bd956ec5 --- /dev/null +++ b/sonar @@ -0,0 +1 @@ +Subproject commit bd956ec52b911168366a657e7140deed8d299b79 diff --git a/tests/unit_tests/integrations/__init__.py b/tests/unit_tests/integrations/__init__.py new file mode 100644 index 00000000..d9db7cad --- /dev/null +++ b/tests/unit_tests/integrations/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Tests for Sonar integrations.""" diff --git a/tests/unit_tests/integrations/test_integration.py b/tests/unit_tests/integrations/test_integration.py new file mode 100644 index 00000000..fafb6638 --- /dev/null +++ b/tests/unit_tests/integrations/test_integration.py @@ -0,0 +1,325 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Tests for Integration class.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from airbyte import exceptions as exc +from airbyte._executors.sonar import SonarExecutor +from airbyte.integrations.base import Integration +from airbyte.integrations.util import get_integration + + +@pytest.fixture +def mock_yaml_path(tmp_path: Path) -> Path: + """Create a mock YAML file.""" + yaml_file = tmp_path / "test-connector.yaml" + yaml_file.write_text(""" +connector: + name: test-api + version: 1.0.0 +resources: + - name: customers + verbs: [list, get, create] + - name: products + verbs: [list, get] +""") + return yaml_file + + +@pytest.fixture +def mock_executor(mock_yaml_path: Path) -> SonarExecutor: + """Create a mock SonarExecutor.""" + return SonarExecutor( + name="test-api", + yaml_path=mock_yaml_path, + secrets={"api_key": "test_key"}, + ) + + +@pytest.fixture +def integration(mock_executor: SonarExecutor, mock_yaml_path: Path) -> Integration: + """Create an Integration instance.""" + return Integration( + executor=mock_executor, + name="test-api", + yaml_path=mock_yaml_path, + ) + + +class TestIntegration: + """Tests for Integration class.""" + + def test_init(self, integration: Integration, mock_yaml_path: Path) -> None: + """Test Integration initialization.""" + assert integration.name == "test-api" + assert integration.yaml_path == mock_yaml_path + assert integration.connector_type == "integration" + + def test_init_with_nonexistent_yaml(self, mock_executor: SonarExecutor) -> None: + """Test Integration initialization with nonexistent YAML.""" + integration = Integration( + executor=mock_executor, + name="test-api", + yaml_path="/nonexistent/path.yaml", + ) + assert integration.yaml_path == Path("/nonexistent/path.yaml") + + @patch("connector_sdk.config_loader.load_connector_config") + def test_validate_yaml_success( + self, + mock_load: MagicMock, + integration: Integration, + ) -> None: + """Test YAML validation success.""" + mock_load.return_value = MagicMock() + integration._validate_yaml() + mock_load.assert_called_once() + + @patch("connector_sdk.config_loader.load_connector_config") + def test_validate_yaml_import_error( + self, + mock_load: MagicMock, + integration: Integration, + ) -> None: + """Test YAML validation with missing connector-sdk.""" + mock_load.side_effect = ImportError("No module named 'connector_sdk'") + + with pytest.raises(exc.PyAirbyteInputError) as excinfo: + integration._validate_yaml() + + assert "connector-sdk is required" in str(excinfo.value) + + @patch("connector_sdk.config_loader.load_connector_config") + def test_validate_yaml_invalid( + self, + mock_load: MagicMock, + integration: Integration, + ) -> None: + """Test YAML validation with invalid YAML.""" + mock_load.side_effect = ValueError("Invalid YAML") + + with pytest.raises(exc.PyAirbyteInputError) as excinfo: + integration._validate_yaml() + + assert "Invalid connector YAML" in str(excinfo.value) + + @patch("connector_sdk.config_loader.load_connector_config") + def test_list_resources( + self, + mock_load: MagicMock, + integration: Integration, + ) -> None: + """Test listing resources.""" + mock_config = MagicMock() + mock_resource1 = MagicMock() + mock_resource1.name = "customers" + mock_resource2 = MagicMock() + mock_resource2.name = "products" + mock_config.resources = [mock_resource1, mock_resource2] + mock_load.return_value = mock_config + + resources = integration.list_resources() + + assert resources == ["customers", "products"] + assert integration._resources == ["customers", "products"] + + @patch("connector_sdk.config_loader.load_connector_config") + def test_list_verbs( + self, + mock_load: MagicMock, + integration: Integration, + ) -> None: + """Test listing verbs for a resource.""" + mock_config = MagicMock() + mock_resource = MagicMock() + mock_resource.name = "customers" + mock_verb1 = MagicMock() + mock_verb1.value = "list" + mock_verb2 = MagicMock() + mock_verb2.value = "get" + mock_resource.verbs = [mock_verb1, mock_verb2] + mock_config.resources = [mock_resource] + mock_load.return_value = mock_config + + verbs = integration.list_verbs("customers") + + assert verbs == ["list", "get"] + + @patch("connector_sdk.config_loader.load_connector_config") + def test_list_verbs_resource_not_found( + self, + mock_load: MagicMock, + integration: Integration, + ) -> None: + """Test listing verbs for nonexistent resource.""" + mock_config = MagicMock() + mock_resource = MagicMock() + mock_resource.name = "customers" + mock_config.resources = [mock_resource] + mock_load.return_value = mock_config + + with pytest.raises(exc.PyAirbyteInputError) as excinfo: + integration.list_verbs("nonexistent") + + assert "Resource 'nonexistent' not found" in str(excinfo.value) + + @pytest.mark.asyncio + async def test_aexecute(self, integration: Integration) -> None: + """Test async execute.""" + mock_response = {"id": "123", "name": "Test"} + + with patch.object( + integration.executor, + "aexecute", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aexecute: + result = await integration.aexecute( + "customers", + "get", + params={"id": "123"}, + ) + + assert result == mock_response + mock_aexecute.assert_called_once_with( + "customers", + "get", + {"id": "123"}, + ) + + def test_execute(self, integration: Integration) -> None: + """Test sync execute.""" + mock_response = {"id": "123", "name": "Test"} + + with patch.object( + integration, + "aexecute", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aexecute: + result = integration.execute( + "customers", + "get", + params={"id": "123"}, + ) + + assert result == mock_response + mock_aexecute.assert_called_once_with( + "customers", + "get", + {"id": "123"}, + ) + + @pytest.mark.asyncio + async def test_aexecute_batch(self, integration: Integration) -> None: + """Test async batch execute.""" + mock_responses = [ + {"id": "1", "name": "Customer 1"}, + {"id": "2", "name": "Customer 2"}, + ] + + with patch.object( + integration.executor, + "aexecute_batch", + new_callable=AsyncMock, + return_value=mock_responses, + ) as mock_batch: + operations = [ + ("customers", "get", {"id": "1"}), + ("customers", "get", {"id": "2"}), + ] + results = await integration.aexecute_batch(operations) + + assert results == mock_responses + mock_batch.assert_called_once_with(operations) + + def test_execute_batch(self, integration: Integration) -> None: + """Test sync batch execute.""" + mock_responses = [ + {"id": "1", "name": "Customer 1"}, + {"id": "2", "name": "Customer 2"}, + ] + + with patch.object( + integration, + "aexecute_batch", + new_callable=AsyncMock, + return_value=mock_responses, + ) as mock_batch: + operations = [ + ("customers", "get", {"id": "1"}), + ("customers", "get", {"id": "2"}), + ] + results = integration.execute_batch(operations) + + assert results == mock_responses + mock_batch.assert_called_once_with(operations) + + def test_check_not_implemented(self, integration: Integration) -> None: + """Test check raises NotImplementedError when no authorize verb.""" + with patch.object(integration, "list_resources", return_value=["customers"]): + with patch.object(integration, "list_verbs", return_value=["list", "get"]): + with pytest.raises(NotImplementedError) as excinfo: + integration.check() + + assert "does not define a health check mechanism" in str(excinfo.value) + + def test_check_with_authorize_verb(self, integration: Integration) -> None: + """Test check succeeds when authorize verb exists.""" + with patch.object(integration, "list_resources", return_value=["auth"]): + with patch.object(integration, "list_verbs", return_value=["authorize"]): + with patch.object( + integration, + "execute", + return_value={"status": "ok"}, + ) as mock_execute: + integration.check() + mock_execute.assert_called_once_with("auth", "authorize", params={}) + + +class TestGetIntegration: + """Tests for get_integration factory function.""" + + def test_get_integration(self, mock_yaml_path: Path) -> None: + """Test get_integration factory.""" + integration = get_integration( + name="test-api", + yaml_path=mock_yaml_path, + secrets={"api_key": "test_key"}, + ) + + assert isinstance(integration, Integration) + assert integration.name == "test-api" + assert integration.yaml_path == mock_yaml_path + assert isinstance(integration.executor, SonarExecutor) + + def test_get_integration_with_logging(self, mock_yaml_path: Path) -> None: + """Test get_integration with logging enabled.""" + integration = get_integration( + name="test-api", + yaml_path=mock_yaml_path, + secrets={"api_key": "test_key"}, + enable_logging=True, + log_file="/tmp/test.log", + ) + + assert isinstance(integration, Integration) + assert integration.executor.enable_logging is True + assert integration.executor.log_file == "/tmp/test.log" + + def test_get_integration_with_validation(self, mock_yaml_path: Path) -> None: + """Test get_integration with validation.""" + with patch("connector_sdk.config_loader.load_connector_config"): + integration = get_integration( + name="test-api", + yaml_path=mock_yaml_path, + secrets={"api_key": "test_key"}, + validate=True, + ) + + assert isinstance(integration, Integration) diff --git a/tests/unit_tests/integrations/test_sonar_executor.py b/tests/unit_tests/integrations/test_sonar_executor.py new file mode 100644 index 00000000..d16ae4b2 --- /dev/null +++ b/tests/unit_tests/integrations/test_sonar_executor.py @@ -0,0 +1,219 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +"""Tests for SonarExecutor class.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from airbyte import exceptions as exc +from airbyte._executors.sonar import SonarExecutor + + +@pytest.fixture +def mock_yaml_path(tmp_path: Path) -> Path: + """Create a mock YAML file.""" + yaml_file = tmp_path / "test-connector.yaml" + yaml_file.write_text(""" +connector: + name: test-api + version: 1.0.0 +resources: + - name: customers + verbs: [list, get] +""") + return yaml_file + + +@pytest.fixture +def executor(mock_yaml_path: Path) -> SonarExecutor: + """Create a SonarExecutor instance.""" + return SonarExecutor( + name="test-api", + yaml_path=mock_yaml_path, + secrets={"api_key": "test_key"}, + ) + + +class TestSonarExecutor: + """Tests for SonarExecutor class.""" + + def test_init(self, executor: SonarExecutor, mock_yaml_path: Path) -> None: + """Test SonarExecutor initialization.""" + assert executor.name == "test-api" + assert executor.yaml_path == mock_yaml_path + assert executor.secrets == {"api_key": "test_key"} + assert executor.enable_logging is False + assert executor.log_file is None + assert executor._executor is None + + def test_init_with_nonexistent_yaml(self) -> None: + """Test SonarExecutor initialization with nonexistent YAML.""" + with pytest.raises(exc.PyAirbyteInputError) as excinfo: + SonarExecutor( + name="test-api", + yaml_path="/nonexistent/path.yaml", + secrets={}, + ) + + assert "Connector YAML file not found" in str(excinfo.value) + + def test_init_with_logging(self, mock_yaml_path: Path) -> None: + """Test SonarExecutor initialization with logging.""" + executor = SonarExecutor( + name="test-api", + yaml_path=mock_yaml_path, + secrets={}, + enable_logging=True, + log_file="/tmp/test.log", + ) + + assert executor.enable_logging is True + assert executor.log_file == "/tmp/test.log" + + @patch("connector_sdk.ConnectorExecutor") + def test_get_executor( + self, + mock_connector_executor: MagicMock, + executor: SonarExecutor, + ) -> None: + """Test lazy executor creation.""" + mock_executor_instance = MagicMock() + mock_connector_executor.return_value = mock_executor_instance + + result = executor._get_executor() + + assert result == mock_executor_instance + assert executor._executor == mock_executor_instance + mock_connector_executor.assert_called_once() + + def test_get_executor_import_error(self, executor: SonarExecutor) -> None: + """Test executor creation with missing connector-sdk.""" + with patch("connector_sdk.ConnectorExecutor") as mock: + mock.side_effect = ImportError("No module named 'connector_sdk'") + + with patch.dict("sys.modules", {"connector_sdk": None}): + with pytest.raises(exc.PyAirbyteInputError) as excinfo: + executor._get_executor() + + assert "connector-sdk is required" in str(excinfo.value) + + def test_cli_property(self, executor: SonarExecutor) -> None: + """Test _cli property returns empty list.""" + assert executor._cli == [] + + def test_execute_not_supported(self, executor: SonarExecutor) -> None: + """Test execute raises NotImplementedError.""" + with pytest.raises(NotImplementedError) as excinfo: + list(executor.execute(["spec"])) + + assert "do not support subprocess execution" in str(excinfo.value) + + @pytest.mark.asyncio + async def test_aexecute(self, executor: SonarExecutor) -> None: + """Test async execute.""" + mock_response = {"id": "123", "name": "Test"} + mock_executor_instance = MagicMock() + mock_executor_instance.execute = AsyncMock(return_value=mock_response) + + with patch.object( + executor, "_get_executor", return_value=mock_executor_instance + ): + result = await executor.aexecute("customers", "get", {"id": "123"}) + + assert result == mock_response + mock_executor_instance.execute.assert_called_once_with( + "customers", + "get", + {"id": "123"}, + ) + + @pytest.mark.asyncio + async def test_aexecute_batch(self, executor: SonarExecutor) -> None: + """Test async batch execute.""" + mock_responses = [{"id": "1"}, {"id": "2"}] + mock_executor_instance = MagicMock() + mock_executor_instance.execute_batch = AsyncMock(return_value=mock_responses) + + with patch.object( + executor, "_get_executor", return_value=mock_executor_instance + ): + operations = [ + ("customers", "get", {"id": "1"}), + ("customers", "get", {"id": "2"}), + ] + results = await executor.aexecute_batch(operations) + + assert results == mock_responses + mock_executor_instance.execute_batch.assert_called_once_with(operations) + + def test_ensure_installation(self, executor: SonarExecutor) -> None: + """Test ensure_installation is a no-op.""" + executor.ensure_installation() # Should not raise + + def test_install(self, executor: SonarExecutor) -> None: + """Test install is a no-op.""" + executor.install() # Should not raise + + def test_uninstall(self, executor: SonarExecutor) -> None: + """Test uninstall is a no-op.""" + executor.uninstall() # Should not raise + + @patch("airbyte._executors.sonar.load_connector_config") + def test_get_installed_version( + self, + mock_load: MagicMock, + executor: SonarExecutor, + ) -> None: + """Test get_installed_version.""" + mock_config = MagicMock() + mock_config.connector.version = "1.0.0" + mock_load.return_value = mock_config + + version = executor.get_installed_version() + + assert version == "1.0.0" + + @patch("airbyte._executors.sonar.load_connector_config") + def test_get_installed_version_no_connector( + self, + mock_load: MagicMock, + executor: SonarExecutor, + ) -> None: + """Test get_installed_version with no connector metadata.""" + mock_config = MagicMock() + mock_config.connector = None + mock_load.return_value = mock_config + + version = executor.get_installed_version() + + assert version is None + + @patch("airbyte._executors.sonar.load_connector_config") + def test_get_installed_version_error( + self, + mock_load: MagicMock, + executor: SonarExecutor, + ) -> None: + """Test get_installed_version with error.""" + mock_load.side_effect = Exception("Load failed") + + version = executor.get_installed_version() + + assert version is None + + @patch("airbyte._executors.sonar.load_connector_config") + def test_get_installed_version_raise_on_error( + self, + mock_load: MagicMock, + executor: SonarExecutor, + ) -> None: + """Test get_installed_version with raise_on_error.""" + mock_load.side_effect = Exception("Load failed") + + with pytest.raises(Exception) as excinfo: + executor.get_installed_version(raise_on_error=True) + + assert "Load failed" in str(excinfo.value)