From dcf910f1bbfd3b805cde8e1b33f4a0990c5a4194 Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sat, 8 Nov 2025 23:20:01 +0300 Subject: [PATCH 01/10] feat(python-sdk): implement task registration and decorator API --- packages/python-sdk/README.md | 46 +++++++ packages/python-sdk/pyproject.toml | 21 ++++ packages/python-sdk/setup.py | 10 ++ .../tests/test_task_registration.py | 100 +++++++++++++++ packages/python-sdk/trigger_sdk/__init__.py | 14 +++ packages/python-sdk/trigger_sdk/py.typed | 0 packages/python-sdk/trigger_sdk/task.py | 115 ++++++++++++++++++ packages/python-sdk/trigger_sdk/types.py | 37 ++++++ packages/python-sdk/trigger_sdk/version.py | 3 + 9 files changed, 346 insertions(+) create mode 100644 packages/python-sdk/README.md create mode 100644 packages/python-sdk/pyproject.toml create mode 100644 packages/python-sdk/setup.py create mode 100644 packages/python-sdk/tests/test_task_registration.py create mode 100644 packages/python-sdk/trigger_sdk/__init__.py create mode 100644 packages/python-sdk/trigger_sdk/py.typed create mode 100644 packages/python-sdk/trigger_sdk/task.py create mode 100644 packages/python-sdk/trigger_sdk/types.py create mode 100644 packages/python-sdk/trigger_sdk/version.py diff --git a/packages/python-sdk/README.md b/packages/python-sdk/README.md new file mode 100644 index 0000000000..7db05068f9 --- /dev/null +++ b/packages/python-sdk/README.md @@ -0,0 +1,46 @@ +# Trigger.dev Python SDK + +Python SDK for Trigger.dev v3 + +## Installation + +```bash +pip install trigger-sdk +``` + +## Quick Start + +```python +from trigger_sdk import task + +@task("my-task-id") +async def my_task(payload): + return {"result": "success"} +``` + +## Features + +- Task registration with decorator API +- Support for both sync and async functions +- Retry configuration +- Queue configuration +- Task duration limits +- Type-safe with Pydantic models + +## Requirements + +- Python >= 3.10 +- pydantic >= 2.0.0 + +## Development + +```bash +# Install development dependencies +pip install -e ".[dev]" + +# Run tests +pytest + +# Run type checking +mypy trigger_sdk +``` diff --git a/packages/python-sdk/pyproject.toml b/packages/python-sdk/pyproject.toml new file mode 100644 index 0000000000..e55dc9f199 --- /dev/null +++ b/packages/python-sdk/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "trigger-sdk" +version = "0.1.0" +description = "Python SDK for Trigger.dev v3" +requires-python = ">=3.10" +dependencies = [ + "pydantic>=2.0.0", + "typing-extensions>=4.5.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "black>=23.0.0", + "mypy>=1.0.0", +] diff --git a/packages/python-sdk/setup.py b/packages/python-sdk/setup.py new file mode 100644 index 0000000000..4d261556bf --- /dev/null +++ b/packages/python-sdk/setup.py @@ -0,0 +1,10 @@ +"""Setup script for trigger-sdk""" + +from setuptools import setup, find_packages + +setup( + packages=find_packages(), + package_data={ + "trigger_sdk": ["py.typed"], + }, +) diff --git a/packages/python-sdk/tests/test_task_registration.py b/packages/python-sdk/tests/test_task_registration.py new file mode 100644 index 0000000000..8aa59a0121 --- /dev/null +++ b/packages/python-sdk/tests/test_task_registration.py @@ -0,0 +1,100 @@ +"""Tests for task registration""" + +import pytest +from trigger_sdk import task, Task, TASK_REGISTRY +from trigger_sdk.task import clear_registry + + +@pytest.fixture(autouse=True) +def reset_registry(): + """Clear registry before each test""" + clear_registry() + yield + clear_registry() + + +def test_task_decorator_sync(): + """Test task decorator with sync function""" + @task("test-sync") + def my_task(payload): + return {"result": payload["value"] * 2} + + assert isinstance(my_task, Task) + assert my_task.id == "test-sync" + assert "test-sync" in TASK_REGISTRY + + +def test_task_decorator_async(): + """Test task decorator with async function""" + @task("test-async") + async def my_task(payload): + return {"result": payload["value"] * 2} + + assert isinstance(my_task, Task) + assert my_task.id == "test-async" + + +@pytest.mark.asyncio +async def test_task_execution_async(): + """Test executing async task""" + @task("test-exec-async") + async def my_task(payload): + return {"result": payload["value"] * 2} + + result = await my_task.execute({"value": 21}) + assert result == {"result": 42} + + +@pytest.mark.asyncio +async def test_task_execution_sync(): + """Test executing sync task""" + @task("test-exec-sync") + def my_task(payload): + return {"result": payload["value"] * 2} + + result = await my_task.execute({"value": 21}) + assert result == {"result": 42} + + +def test_task_with_retry_config(): + """Test task with retry configuration""" + @task("test-retry", retry={"maxAttempts": 5, "factor": 3.0}) + async def my_task(payload): + return payload + + assert my_task.config.retry.maxAttempts == 5 + assert my_task.config.retry.factor == 3.0 + + +def test_task_with_queue_config(): + """Test task with queue configuration""" + @task("test-queue", queue={"name": "critical", "concurrencyLimit": 10}) + async def my_task(payload): + return payload + + assert my_task.config.queue.name == "critical" + assert my_task.config.queue.concurrencyLimit == 10 + + +def test_duplicate_task_id_raises(): + """Test that duplicate task IDs raise an error""" + @task("duplicate-id") + async def task1(payload): + return payload + + with pytest.raises(ValueError, match="already registered"): + @task("duplicate-id") + async def task2(payload): + return payload + + +def test_get_task_metadata(): + """Test task metadata generation""" + @task("test-metadata", max_duration=300) + async def my_task(payload): + return payload + + metadata = my_task.get_metadata() + assert metadata.id == "test-metadata" + assert metadata.exportName == "test-metadata" + assert metadata.maxDuration == 300 diff --git a/packages/python-sdk/trigger_sdk/__init__.py b/packages/python-sdk/trigger_sdk/__init__.py new file mode 100644 index 0000000000..3977826129 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/__init__.py @@ -0,0 +1,14 @@ +"""Trigger.dev Python SDK v3""" + +from trigger_sdk.task import task, Task, TASK_REGISTRY +from trigger_sdk.types import TaskConfig, RetryConfig, QueueConfig + +__version__ = "0.1.0" +__all__ = [ + "task", + "Task", + "TASK_REGISTRY", + "TaskConfig", + "RetryConfig", + "QueueConfig", +] diff --git a/packages/python-sdk/trigger_sdk/py.typed b/packages/python-sdk/trigger_sdk/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/python-sdk/trigger_sdk/task.py b/packages/python-sdk/trigger_sdk/task.py new file mode 100644 index 0000000000..dac8f096ff --- /dev/null +++ b/packages/python-sdk/trigger_sdk/task.py @@ -0,0 +1,115 @@ +"""Task decorator and registration""" + +import asyncio +import inspect +from typing import Any, Callable, Dict, Optional, TypeVar, Union +from trigger_sdk.types import TaskConfig, TaskMetadata, RetryConfig, QueueConfig + +# Global task registry +TASK_REGISTRY: Dict[str, "Task"] = {} + +TPayload = TypeVar("TPayload") +TOutput = TypeVar("TOutput") + + +class Task: + """Represents a registered task""" + + def __init__( + self, + config: TaskConfig, + run_fn: Callable[[Any], Any], + file_path: Optional[str] = None, + ): + self.id = config.id + self.config = config + self.run_fn = run_fn + self.file_path = file_path or "" + + # Validate function signature + if not (inspect.iscoroutinefunction(run_fn) or inspect.isfunction(run_fn)): + raise TypeError(f"Task '{self.id}' must be a function or async function") + + # Register task + if self.id in TASK_REGISTRY: + raise ValueError(f"Task with id '{self.id}' already registered") + + TASK_REGISTRY[self.id] = self + + async def execute(self, payload: Any) -> Any: + """Execute the task with the given payload""" + if inspect.iscoroutinefunction(self.run_fn): + return await self.run_fn(payload) + else: + # Run sync function in executor to avoid blocking + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self.run_fn, payload) + + def get_metadata(self) -> TaskMetadata: + """Get task metadata for indexing""" + return TaskMetadata( + id=self.id, + filePath=self.file_path, + exportName=self.id, # Python uses ID as export name + retry=self.config.retry, + queue=self.config.queue, + maxDuration=self.config.maxDuration, + ) + + def __repr__(self) -> str: + return f"Task(id='{self.id}', file='{self.file_path}')" + + +def task( + id: str, + *, + retry: Optional[Union[RetryConfig, Dict[str, Any]]] = None, + queue: Optional[Union[QueueConfig, Dict[str, Any]]] = None, + max_duration: Optional[int] = None, +) -> Callable[[Callable[..., Any]], Task]: + """ + Decorator to register a task. + + Usage: + @task("my-task-id", retry={"maxAttempts": 5}) + async def my_task(payload): + return {"result": "success"} + + Args: + id: Unique task identifier + retry: Retry configuration (dict or RetryConfig) + queue: Queue configuration (dict or QueueConfig) + max_duration: Maximum task duration in seconds + + Returns: + Decorated task function as a Task instance + """ + # Convert dict to models if needed (with None check fix) + retry_config = RetryConfig(**retry) if isinstance(retry, dict) else retry + queue_config = QueueConfig(**queue) if isinstance(queue, dict) else queue + + config = TaskConfig( + id=id, + retry=retry_config, + queue=queue_config, + maxDuration=max_duration, + ) + + def decorator(fn: Callable[..., Any]) -> Task: + # Get file path from function + file_path = inspect.getfile(fn) if hasattr(fn, "__code__") else None + + task_obj = Task(config=config, run_fn=fn, file_path=file_path) + return task_obj + + return decorator + + +def get_all_tasks() -> Dict[str, Task]: + """Get all registered tasks""" + return TASK_REGISTRY.copy() + + +def clear_registry() -> None: + """Clear the task registry (useful for testing)""" + TASK_REGISTRY.clear() diff --git a/packages/python-sdk/trigger_sdk/types.py b/packages/python-sdk/trigger_sdk/types.py new file mode 100644 index 0000000000..75fad3c05b --- /dev/null +++ b/packages/python-sdk/trigger_sdk/types.py @@ -0,0 +1,37 @@ +"""Type definitions for the Python SDK""" + +from typing import Optional +from pydantic import BaseModel + + +class RetryConfig(BaseModel): + """Task retry configuration""" + maxAttempts: int = 3 + minTimeoutInMs: int = 1000 + maxTimeoutInMs: int = 60000 + factor: float = 2.0 + randomize: bool = True + + +class QueueConfig(BaseModel): + """Task queue configuration""" + name: Optional[str] = None + concurrencyLimit: Optional[int] = None + + +class TaskConfig(BaseModel): + """Task configuration""" + id: str + retry: Optional[RetryConfig] = None + queue: Optional[QueueConfig] = None + maxDuration: Optional[int] = None # seconds + + +class TaskMetadata(BaseModel): + """Task metadata for registration""" + id: str + filePath: str + exportName: str + retry: Optional[RetryConfig] = None + queue: Optional[QueueConfig] = None + maxDuration: Optional[int] = None diff --git a/packages/python-sdk/trigger_sdk/version.py b/packages/python-sdk/trigger_sdk/version.py new file mode 100644 index 0000000000..9304a2ffb0 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/version.py @@ -0,0 +1,3 @@ +"""Version information for trigger-sdk""" + +__version__ = "0.1.0" From 6eb8dac710347fcbc3215ecfaba4744620f86d55 Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sat, 8 Nov 2025 23:45:20 +0300 Subject: [PATCH 02/10] feat(python-sdk): implement IPC layer with stdio transport - Add Pydantic schemas matching TypeScript message types - Create transport-agnostic IpcConnection interface - Implement StdioIpcConnection for line-delimited JSON communication --- .../python-sdk/tests/test_error_mapping.py | 239 ++++++++++++ packages/python-sdk/tests/test_ipc_stdio.py | 326 +++++++++++++++++ packages/python-sdk/tests/test_schemas.py | 344 ++++++++++++++++++ packages/python-sdk/trigger_sdk/__init__.py | 13 + packages/python-sdk/trigger_sdk/errors.py | 119 ++++++ .../python-sdk/trigger_sdk/ipc/__init__.py | 11 + packages/python-sdk/trigger_sdk/ipc/base.py | 169 +++++++++ packages/python-sdk/trigger_sdk/ipc/stdio.py | 134 +++++++ .../trigger_sdk/schemas/__init__.py | 86 +++++ .../python-sdk/trigger_sdk/schemas/common.py | 138 +++++++ .../python-sdk/trigger_sdk/schemas/errors.py | 68 ++++ .../trigger_sdk/schemas/messages.py | 130 +++++++ .../trigger_sdk/schemas/resources.py | 46 +++ 13 files changed, 1823 insertions(+) create mode 100644 packages/python-sdk/tests/test_error_mapping.py create mode 100644 packages/python-sdk/tests/test_ipc_stdio.py create mode 100644 packages/python-sdk/tests/test_schemas.py create mode 100644 packages/python-sdk/trigger_sdk/errors.py create mode 100644 packages/python-sdk/trigger_sdk/ipc/__init__.py create mode 100644 packages/python-sdk/trigger_sdk/ipc/base.py create mode 100644 packages/python-sdk/trigger_sdk/ipc/stdio.py create mode 100644 packages/python-sdk/trigger_sdk/schemas/__init__.py create mode 100644 packages/python-sdk/trigger_sdk/schemas/common.py create mode 100644 packages/python-sdk/trigger_sdk/schemas/errors.py create mode 100644 packages/python-sdk/trigger_sdk/schemas/messages.py create mode 100644 packages/python-sdk/trigger_sdk/schemas/resources.py diff --git a/packages/python-sdk/tests/test_error_mapping.py b/packages/python-sdk/tests/test_error_mapping.py new file mode 100644 index 0000000000..ee735fc9df --- /dev/null +++ b/packages/python-sdk/tests/test_error_mapping.py @@ -0,0 +1,239 @@ +"""Tests for exception to TaskRunError conversion""" + +import json +import pytest +from trigger_sdk.errors import exception_to_task_run_error, get_error_code_for_exception +from trigger_sdk.schemas.errors import ( + TaskRunBuiltInError, + TaskRunInternalError, + TaskRunStringError, +) + + +class TestBuiltInErrorMapping: + """Test mapping of built-in Python exceptions""" + + def test_type_error_to_built_in_error(self): + """Test TypeError maps to BUILT_IN_ERROR""" + exc = TypeError("cannot add str and int") + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunBuiltInError) + assert error.type == "BUILT_IN_ERROR" + assert error.name == "TypeError" + assert "cannot add str and int" in error.message + assert len(error.stackTrace) > 0 + + def test_value_error_to_built_in_error(self): + """Test ValueError maps to BUILT_IN_ERROR""" + exc = ValueError("invalid literal for int()") + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunBuiltInError) + assert error.name == "ValueError" + assert "invalid literal" in error.message + + def test_attribute_error_to_built_in_error(self): + """Test AttributeError maps to BUILT_IN_ERROR""" + exc = AttributeError("object has no attribute 'foo'") + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunBuiltInError) + assert error.name == "AttributeError" + + def test_key_error_to_built_in_error(self): + """Test KeyError maps to BUILT_IN_ERROR""" + exc = KeyError("missing_key") + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunBuiltInError) + assert error.name == "KeyError" + + def test_runtime_error_to_built_in_error(self): + """Test RuntimeError maps to BUILT_IN_ERROR""" + exc = RuntimeError("something went wrong") + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunBuiltInError) + assert error.name == "RuntimeError" + + +class TestInternalErrorMapping: + """Test mapping of system errors to INTERNAL_ERROR""" + + def test_import_error_to_internal_error(self): + """Test ImportError maps to INTERNAL_ERROR with COULD_NOT_IMPORT_TASK""" + exc = ImportError("No module named 'missing_module'") + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunInternalError) + assert error.type == "INTERNAL_ERROR" + assert error.code == "COULD_NOT_IMPORT_TASK" + assert "missing_module" in error.message + + def test_module_not_found_error_to_internal_error(self): + """Test ModuleNotFoundError maps to INTERNAL_ERROR""" + exc = ModuleNotFoundError("No module named 'foo'") + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunInternalError) + assert error.code == "COULD_NOT_IMPORT_TASK" + + def test_system_exit_to_internal_error(self): + """Test SystemExit maps to TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE""" + exc = SystemExit(1) + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunInternalError) + assert error.code == "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE" + + def test_keyboard_interrupt_to_cancelled(self): + """Test KeyboardInterrupt maps to TASK_RUN_CANCELLED""" + exc = KeyboardInterrupt() + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunInternalError) + assert error.code == "TASK_RUN_CANCELLED" + + def test_syntax_error_to_input_error(self): + """Test SyntaxError maps to TASK_INPUT_ERROR""" + exc = SyntaxError("invalid syntax") + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunInternalError) + assert error.code == "TASK_INPUT_ERROR" + + def test_timeout_error_to_max_duration(self): + """Test TimeoutError maps to MAX_DURATION_EXCEEDED""" + exc = TimeoutError("Task timed out") + error = exception_to_task_run_error(exc) + + # Could be BUILT_IN_ERROR or INTERNAL_ERROR depending on context + # Check that it's handled properly + if isinstance(error, TaskRunInternalError): + assert error.code == "MAX_DURATION_EXCEEDED" + else: + # TimeoutError is also a built-in, so this is acceptable + assert isinstance(error, TaskRunBuiltInError) + + +class TestErrorCodeMapping: + """Test get_error_code_for_exception() function""" + + def test_import_error_code(self): + """Test ImportError returns correct code""" + code = get_error_code_for_exception(ImportError()) + assert code == "COULD_NOT_IMPORT_TASK" + + def test_system_exit_code(self): + """Test SystemExit returns correct code""" + code = get_error_code_for_exception(SystemExit(1)) + assert code == "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE" + + def test_keyboard_interrupt_code(self): + """Test KeyboardInterrupt returns correct code""" + code = get_error_code_for_exception(KeyboardInterrupt()) + assert code == "TASK_RUN_CANCELLED" + + def test_syntax_error_code(self): + """Test SyntaxError returns correct code""" + code = get_error_code_for_exception(SyntaxError()) + assert code == "TASK_INPUT_ERROR" + + def test_generic_exception_code(self): + """Test generic Exception returns fallback code""" + code = get_error_code_for_exception(Exception()) + assert code == "TASK_EXECUTION_FAILED" + + +class TestStackTracePreservation: + """Test that stack traces are preserved in error conversion""" + + def test_stack_trace_captured(self): + """Test that stack trace is included in error""" + try: + raise ValueError("test error") + except ValueError as e: + error = exception_to_task_run_error(e) + + assert len(error.stackTrace) > 0 + assert "test error" in error.stackTrace + assert "Traceback" in error.stackTrace + + def test_nested_stack_trace(self): + """Test stack trace with nested calls""" + def inner(): + raise RuntimeError("inner error") + + def outer(): + inner() + + try: + outer() + except RuntimeError as e: + error = exception_to_task_run_error(e) + + assert isinstance(error, TaskRunBuiltInError) + assert "inner" in error.stackTrace + assert "outer" in error.stackTrace + + +class TestStringErrorFallback: + """Test fallback to STRING_ERROR for unknown exceptions""" + + def test_custom_exception_to_string_error(self): + """Test that custom exceptions without mapping become STRING_ERROR""" + class CustomException(Exception): + pass + + exc = CustomException("custom error message") + error = exception_to_task_run_error(exc) + + # Custom exception should have error code fallback + assert isinstance(error, TaskRunInternalError) + assert error.code == "TASK_EXECUTION_FAILED" + + def test_error_serialization(self): + """Test that all error types can be serialized to JSON""" + errors = [ + exception_to_task_run_error(ValueError("test")), + exception_to_task_run_error(ImportError("test")), + exception_to_task_run_error(KeyboardInterrupt()), + ] + + for error in errors: + # Should serialize without errors + json_str = error.model_dump_json() + data = json.loads(json_str) + + # Should have type field + assert "type" in data + assert data["type"] in ["BUILT_IN_ERROR", "INTERNAL_ERROR", "STRING_ERROR"] + + +class TestErrorMessageFormatting: + """Test error message formatting""" + + def test_error_message_contains_exception_text(self): + """Test that error message includes the exception text""" + exc = ValueError("invalid value: 42") + error = exception_to_task_run_error(exc) + + assert "invalid value: 42" in error.message + + def test_error_preserves_exception_name(self): + """Test that error preserves the exception class name""" + exc = ZeroDivisionError("division by zero") + error = exception_to_task_run_error(exc) + + assert isinstance(error, TaskRunBuiltInError) + assert error.name == "ZeroDivisionError" + + def test_empty_exception_message(self): + """Test handling of exceptions with no message""" + exc = ValueError() + error = exception_to_task_run_error(exc) + + # Should still work, just with empty/default message + assert isinstance(error, TaskRunBuiltInError) + assert error.name == "ValueError" diff --git a/packages/python-sdk/tests/test_ipc_stdio.py b/packages/python-sdk/tests/test_ipc_stdio.py new file mode 100644 index 0000000000..22fa3db1d9 --- /dev/null +++ b/packages/python-sdk/tests/test_ipc_stdio.py @@ -0,0 +1,326 @@ +"""Tests for stdio IPC implementation""" + +import json +import asyncio +import pytest +from io import StringIO +from unittest.mock import patch, AsyncMock + +from trigger_sdk.ipc import StdioIpcConnection +from trigger_sdk.schemas import ( + TaskHeartbeatMessage, + ExecuteTaskRunMessage, + CancelMessage, +) + + +class TestStdioIpcSend: + """Test sending messages via stdio""" + + @pytest.mark.asyncio + async def test_send_writes_json_to_stdout(self): + """Test that send() writes line-delimited JSON to stdout""" + ipc = StdioIpcConnection() + message = TaskHeartbeatMessage(id="run_123") + + with patch("sys.stdout", new=StringIO()) as mock_stdout: + await ipc.send(message) + + output = mock_stdout.getvalue() + lines = output.strip().split("\n") + + assert len(lines) == 1 + data = json.loads(lines[0]) + assert data["type"] == "TASK_HEARTBEAT" + assert data["id"] == "run_123" + + @pytest.mark.asyncio + async def test_send_completed_helper(self): + """Test send_completed() convenience method""" + ipc = StdioIpcConnection() + + with patch("sys.stdout", new=StringIO()) as mock_stdout: + await ipc.send_completed( + id="run_123", + output={"result": "success"}, + usage={"durationMs": 1500}, + ) + + output = mock_stdout.getvalue() + data = json.loads(output.strip()) + + assert data["type"] == "TASK_RUN_COMPLETED" + assert data["completion"]["ok"] is True + assert data["completion"]["id"] == "run_123" + assert "result" in json.loads(data["completion"]["output"]) + assert data["completion"]["usage"]["durationMs"] == 1500 + + @pytest.mark.asyncio + async def test_send_failed_helper(self): + """Test send_failed() convenience method""" + ipc = StdioIpcConnection() + error = ValueError("Test error") + + with patch("sys.stdout", new=StringIO()) as mock_stdout: + await ipc.send_failed( + id="run_123", + error=error, + usage={"durationMs": 500}, + ) + + output = mock_stdout.getvalue() + data = json.loads(output.strip()) + + assert data["type"] == "TASK_RUN_FAILED_TO_RUN" + assert data["completion"]["ok"] is False + assert data["completion"]["id"] == "run_123" + assert "Test error" in str(data["completion"]["error"]) + + @pytest.mark.asyncio + async def test_send_heartbeat_helper(self): + """Test send_heartbeat() convenience method""" + ipc = StdioIpcConnection() + + with patch("sys.stdout", new=StringIO()) as mock_stdout: + await ipc.send_heartbeat("run_123") + + output = mock_stdout.getvalue() + data = json.loads(output.strip()) + + assert data["type"] == "TASK_HEARTBEAT" + assert data["id"] == "run_123" + + +class TestStdioIpcReceive: + """Test receiving messages via stdio""" + + @pytest.mark.asyncio + async def test_receive_execute_message(self): + """Test receiving EXECUTE_TASK_RUN message""" + ipc = StdioIpcConnection() + received_messages = [] + + async def handler(message): + received_messages.append(message) + + ipc.on("EXECUTE_TASK_RUN", handler) + + # Create test message + test_message = { + "type": "EXECUTE_TASK_RUN", + "version": "v1", + "execution": { + "task": {"id": "test-task", "filePath": "/task.py"}, + "run": { + "id": "run_123", + "payload": "{}", + "payloadType": "application/json", + "tags": [], + "isTest": False, + }, + "attempt": { + "id": "attempt_123", + "number": 1, + "startedAt": "2024-01-01T00:00:00Z", + }, + }, + } + + # Simulate stdin + stdin_data = json.dumps(test_message) + "\n" + with patch("sys.stdin", StringIO(stdin_data)): + # Start listening (will read one message then hit EOF) + await ipc.start_listening() + + assert len(received_messages) == 1 + assert isinstance(received_messages[0], ExecuteTaskRunMessage) + assert received_messages[0].type == "EXECUTE_TASK_RUN" + + @pytest.mark.asyncio + async def test_receive_cancel_message(self): + """Test receiving CANCEL message""" + ipc = StdioIpcConnection() + received_messages = [] + + def handler(message): + received_messages.append(message) + + ipc.on("CANCEL", handler) + + stdin_data = '{"type": "CANCEL", "version": "v1"}\n' + with patch("sys.stdin", StringIO(stdin_data)): + await ipc.start_listening() + + assert len(received_messages) == 1 + assert isinstance(received_messages[0], CancelMessage) + + @pytest.mark.asyncio + async def test_malformed_json_logged_not_crash(self): + """Test that malformed JSON is logged to stderr and doesn't crash""" + ipc = StdioIpcConnection() + received_messages = [] + + ipc.on("TEST", lambda msg: received_messages.append(msg)) + + # Invalid JSON + stdin_data = '{invalid json}\n' + + with patch("sys.stdin", StringIO(stdin_data)): + with patch("sys.stderr", new=StringIO()) as mock_stderr: + await ipc.start_listening() + + stderr_output = mock_stderr.getvalue() + assert "Invalid JSON" in stderr_output or "JSONDecodeError" in stderr_output + + # Should not have crashed, should have no messages + assert len(received_messages) == 0 + + @pytest.mark.asyncio + async def test_missing_type_field_logged(self): + """Test that messages without 'type' field are logged""" + ipc = StdioIpcConnection() + + stdin_data = '{"version": "v1"}\n' # Missing type field + + with patch("sys.stdin", StringIO(stdin_data)): + with patch("sys.stderr", new=StringIO()) as mock_stderr: + await ipc.start_listening() + + stderr_output = mock_stderr.getvalue() + assert "missing 'type'" in stderr_output.lower() + + @pytest.mark.asyncio + async def test_unknown_message_type_logged(self): + """Test that unknown message types are logged""" + ipc = StdioIpcConnection() + + stdin_data = '{"type": "UNKNOWN_MESSAGE", "version": "v1"}\n' + + with patch("sys.stdin", StringIO(stdin_data)): + with patch("sys.stderr", new=StringIO()) as mock_stderr: + # Will fail validation, should be logged + await ipc.start_listening() + + stderr_output = mock_stderr.getvalue() + # Should have validation error or no handler message + assert len(stderr_output) > 0 + + +class TestStdioIpcHandlers: + """Test message handler registration and dispatch""" + + @pytest.mark.asyncio + async def test_handler_registration(self): + """Test registering multiple handlers""" + ipc = StdioIpcConnection() + + handler1_called = [] + handler2_called = [] + + ipc.on("CANCEL", lambda msg: handler1_called.append(msg)) + ipc.on("FLUSH", lambda msg: handler2_called.append(msg)) + + stdin_data = '{"type": "CANCEL", "version": "v1"}\n{"type": "FLUSH", "version": "v1"}\n' + + with patch("sys.stdin", StringIO(stdin_data)): + await ipc.start_listening() + + assert len(handler1_called) == 1 + assert len(handler2_called) == 1 + + @pytest.mark.asyncio + async def test_async_handler_support(self): + """Test that async handlers are awaited""" + ipc = StdioIpcConnection() + received = [] + + async def async_handler(message): + await asyncio.sleep(0.01) # Simulate async work + received.append(message) + + ipc.on("CANCEL", async_handler) + + stdin_data = '{"type": "CANCEL", "version": "v1"}\n' + with patch("sys.stdin", StringIO(stdin_data)): + await ipc.start_listening() + + assert len(received) == 1 + + @pytest.mark.asyncio + async def test_handler_exception_logged_not_crash(self): + """Test that handler exceptions are caught and logged""" + ipc = StdioIpcConnection() + + def failing_handler(message): + raise RuntimeError("Handler failed") + + ipc.on("CANCEL", failing_handler) + + stdin_data = '{"type": "CANCEL", "version": "v1"}\n' + + with patch("sys.stdin", StringIO(stdin_data)): + with patch("sys.stderr", new=StringIO()) as mock_stderr: + # Should not raise, should log error + await ipc.start_listening() + + stderr_output = mock_stderr.getvalue() + assert "Handler error" in stderr_output or "Handler failed" in stderr_output + + +class TestStdioIpcLifecycle: + """Test IPC connection lifecycle""" + + def test_initial_state(self): + """Test IPC starts in stopped state""" + ipc = StdioIpcConnection() + assert ipc._running is False + + @pytest.mark.asyncio + async def test_stop_method(self): + """Test stop() method sets running to False""" + ipc = StdioIpcConnection() + + # Mock stdin to avoid pytest capture issues + with patch("sys.stdin", StringIO("")): + # Start listening in background + listen_task = asyncio.create_task(ipc.start_listening()) + + # Give it time to start + await asyncio.sleep(0.01) + + # Stop it + ipc.stop() + + # Should finish quickly + await asyncio.wait_for(listen_task, timeout=1.0) + + assert ipc._running is False + + +class TestStdioIpcThreadSafety: + """Test thread-safety of message sending""" + + @pytest.mark.asyncio + async def test_concurrent_sends_not_interleaved(self): + """Test that concurrent sends don't interleave JSON""" + ipc = StdioIpcConnection() + + async def send_message(msg_id): + message = TaskHeartbeatMessage(id=f"run_{msg_id}") + await ipc.send(message) + + with patch("sys.stdout", new=StringIO()) as mock_stdout: + # Send 10 messages concurrently + await asyncio.gather(*[send_message(i) for i in range(10)]) + + output = mock_stdout.getvalue() + lines = [line for line in output.split("\n") if line.strip()] + + # Should have 10 valid JSON lines + assert len(lines) == 10 + + # Each line should be valid JSON + for line in lines: + data = json.loads(line) + assert data["type"] == "TASK_HEARTBEAT" + assert "run_" in data["id"] diff --git a/packages/python-sdk/tests/test_schemas.py b/packages/python-sdk/tests/test_schemas.py new file mode 100644 index 0000000000..d2c7eed079 --- /dev/null +++ b/packages/python-sdk/tests/test_schemas.py @@ -0,0 +1,344 @@ +"""Tests for Pydantic message schemas""" + +import json +import pytest +from trigger_sdk.schemas import ( + # Common + TaskRunExecutionUsage, + TaskInfo, + RunInfo, + AttemptInfo, + TaskRunExecution, + TaskRunSuccessfulExecutionResult, + TaskRunFailedExecutionResult, + # Errors + TaskRunBuiltInError, + TaskRunInternalError, + TaskRunStringError, + # Messages + TaskRunCompletedMessage, + TaskRunFailedMessage, + TaskHeartbeatMessage, + IndexTasksCompleteMessage, + ExecuteTaskRunMessage, + CancelMessage, + FlushMessage, + # Resources + TaskResource, + QueueConfig, + RetryConfig, +) + + +class TestCommonSchemas: + """Test core execution type schemas""" + + def test_task_info_creation(self): + task = TaskInfo(id="test-task", filePath="/path/to/task.py") + assert task.id == "test-task" + assert task.filePath == "/path/to/task.py" + + def test_run_info_defaults(self): + run = RunInfo( + id="run_123", + payload='{"key": "value"}', + payloadType="application/json", + ) + assert run.id == "run_123" + assert run.tags == [] + assert run.isTest is False + + def test_task_run_execution_minimal(self): + """Test with only essential fields""" + execution = TaskRunExecution( + task=TaskInfo(id="task1", filePath="/task.py"), + run=RunInfo(id="run1", payload="{}", payloadType="application/json"), + attempt=AttemptInfo(id="attempt1", number=1, startedAt="2024-01-01T00:00:00Z"), + ) + assert execution.task.id == "task1" + assert execution.run.id == "run1" + assert execution.attempt.number == 1 + # Optional fields should be None + assert execution.organization is None + assert execution.project is None + + def test_successful_execution_result(self): + result = TaskRunSuccessfulExecutionResult( + id="run_123", + output='{"result": "success"}', + ) + assert result.ok is True + assert result.id == "run_123" + assert result.outputType == "application/json" # Default + + def test_failed_execution_result(self): + error = TaskRunBuiltInError( + name="ValueError", + message="Invalid input", + stackTrace="Traceback...", + ) + result = TaskRunFailedExecutionResult( + id="run_123", + error=error, + ) + assert result.ok is False + assert result.id == "run_123" + assert result.error.type == "BUILT_IN_ERROR" + + +class TestErrorSchemas: + """Test error type schemas""" + + def test_built_in_error(self): + error = TaskRunBuiltInError( + name="TypeError", + message="Cannot add str and int", + stackTrace="Traceback (most recent call last):\n ...", + ) + assert error.type == "BUILT_IN_ERROR" + assert error.name == "TypeError" + + def test_internal_error(self): + error = TaskRunInternalError( + code="TASK_EXECUTION_FAILED", + message="Task crashed", + stackTrace="Traceback...", + ) + assert error.type == "INTERNAL_ERROR" + assert error.code == "TASK_EXECUTION_FAILED" + + def test_string_error(self): + error = TaskRunStringError(raw="Something went wrong") + assert error.type == "STRING_ERROR" + assert error.raw == "Something went wrong" + + def test_error_serialization(self): + """Test that errors serialize to JSON correctly""" + error = TaskRunBuiltInError( + name="ValueError", + message="Test error", + stackTrace="", + ) + json_str = error.model_dump_json() + data = json.loads(json_str) + assert data["type"] == "BUILT_IN_ERROR" + assert data["name"] == "ValueError" + + +class TestWorkerMessages: + """Test worker → coordinator messages""" + + def test_task_run_completed_message(self): + result = TaskRunSuccessfulExecutionResult( + id="run_123", + output='{"result": "done"}', + ) + message = TaskRunCompletedMessage.from_result(result) + + assert message.type == "TASK_RUN_COMPLETED" + assert message.version == "v1" + assert message.completion["ok"] is True + assert message.completion["id"] == "run_123" + + def test_task_run_failed_message(self): + error = TaskRunInternalError( + code="TASK_EXECUTION_FAILED", + message="Failed", + ) + result = TaskRunFailedExecutionResult( + id="run_123", + error=error, + ) + message = TaskRunFailedMessage.from_result(result) + + assert message.type == "TASK_RUN_FAILED_TO_RUN" + assert message.version == "v1" + assert message.completion["ok"] is False + + def test_heartbeat_message(self): + message = TaskHeartbeatMessage(id="run_123") + + assert message.type == "TASK_HEARTBEAT" + assert message.version == "v1" + assert message.id == "run_123" + + def test_index_tasks_complete_message(self): + tasks = [ + {"id": "task1", "filePath": "/task1.py", "exportName": "task1"}, + {"id": "task2", "filePath": "/task2.py", "exportName": "task2"}, + ] + message = IndexTasksCompleteMessage(tasks=tasks) + + assert message.type == "INDEX_TASKS_COMPLETE" + assert message.version == "v1" + assert len(message.tasks) == 2 + + +class TestCoordinatorMessages: + """Test coordinator → worker messages""" + + def test_execute_task_run_message(self): + execution_data = { + "task": {"id": "test-task", "filePath": "/task.py"}, + "run": { + "id": "run_123", + "payload": "{}", + "payloadType": "application/json", + "tags": [], + "isTest": False, + }, + "attempt": { + "id": "attempt_123", + "number": 1, + "startedAt": "2024-01-01T00:00:00Z", + }, + } + message = ExecuteTaskRunMessage(execution=execution_data) + + assert message.type == "EXECUTE_TASK_RUN" + assert message.version == "v1" + + # Test parsing execution + execution = message.get_execution() + assert isinstance(execution, TaskRunExecution) + assert execution.task.id == "test-task" + + def test_cancel_message(self): + message = CancelMessage() + assert message.type == "CANCEL" + assert message.version == "v1" + + def test_flush_message(self): + message = FlushMessage() + assert message.type == "FLUSH" + assert message.version == "v1" + + +class TestResourceSchemas: + """Test task resource schemas""" + + def test_task_resource_minimal(self): + resource = TaskResource( + id="test-task", + filePath="/path/to/task.py", + exportName="test-task", + ) + assert resource.id == "test-task" + assert resource.filePath == "/path/to/task.py" + assert resource.exportName == "test-task" + assert resource.description is None + + def test_task_resource_with_configs(self): + resource = TaskResource( + id="test-task", + filePath="/task.py", + exportName="test-task", + queue=QueueConfig(name="critical", concurrencyLimit=5), + retry=RetryConfig(maxAttempts=3, factor=2.0), + maxDuration=300, + ) + assert resource.queue.name == "critical" + assert resource.queue.concurrencyLimit == 5 + assert resource.retry.maxAttempts == 3 + assert resource.maxDuration == 300 + + +class TestMessageSerialization: + """Test JSON serialization/deserialization of messages""" + + def test_worker_message_round_trip(self): + """Test that messages can be serialized and deserialized""" + message = TaskHeartbeatMessage(id="run_123") + + # Serialize to JSON + json_str = message.model_dump_json() + + # Deserialize back + data = json.loads(json_str) + message2 = TaskHeartbeatMessage.model_validate(data) + + assert message2.type == message.type + assert message2.id == message.id + + def test_coordinator_message_from_json(self): + """Test parsing coordinator message from JSON string""" + json_str = '{"type": "CANCEL", "version": "v1"}' + data = json.loads(json_str) + message = CancelMessage.model_validate(data) + + assert message.type == "CANCEL" + assert message.version == "v1" + + def test_execution_result_serialization(self): + """Test full execution result serialization""" + result = TaskRunSuccessfulExecutionResult( + id="run_123", + output='{"data": "test"}', + usage=TaskRunExecutionUsage(durationMs=1500), + ) + + # Serialize + data = result.model_dump() + + # Verify structure + assert data["ok"] is True + assert data["id"] == "run_123" + assert data["outputType"] == "application/json" + assert data["usage"]["durationMs"] == 1500 + + +class TestSchemaDefaults: + """Test that schema defaults match TypeScript expectations""" + + def test_message_version_defaults(self): + """All messages should default to version v1""" + assert TaskHeartbeatMessage(id="test").version == "v1" + assert CancelMessage().version == "v1" + assert FlushMessage().version == "v1" + + def test_output_type_default(self): + """Output type should default to application/json""" + result = TaskRunSuccessfulExecutionResult(id="run_123") + assert result.outputType == "application/json" + + def test_error_type_literals(self): + """Error types should have correct literal values""" + built_in = TaskRunBuiltInError( + name="Error", message="msg", stackTrace="" + ) + internal = TaskRunInternalError(code="INTERNAL_ERROR") + string_err = TaskRunStringError(raw="error") + + assert built_in.type == "BUILT_IN_ERROR" + assert internal.type == "INTERNAL_ERROR" + assert string_err.type == "STRING_ERROR" + + +class TestOptionalFields: + """Test optional field handling""" + + def test_task_run_execution_optional_fields(self): + """Optional fields should not cause validation errors""" + execution = TaskRunExecution( + task=TaskInfo(id="t1", filePath="/t.py"), + run=RunInfo(id="r1", payload="{}", payloadType="json"), + attempt=AttemptInfo(id="a1", number=1, startedAt="2024-01-01"), + ) + + # Should serialize without errors even with None optional fields + data = execution.model_dump() + assert "organization" in data + assert data["organization"] is None + + def test_task_resource_optional_configs(self): + """Task resource should work without queue/retry configs""" + resource = TaskResource( + id="task1", + filePath="/task.py", + exportName="task1", + ) + + data = resource.model_dump() + assert data["queue"] is None + assert data["retry"] is None + assert data["maxDuration"] is None diff --git a/packages/python-sdk/trigger_sdk/__init__.py b/packages/python-sdk/trigger_sdk/__init__.py index 3977826129..015341a723 100644 --- a/packages/python-sdk/trigger_sdk/__init__.py +++ b/packages/python-sdk/trigger_sdk/__init__.py @@ -2,13 +2,26 @@ from trigger_sdk.task import task, Task, TASK_REGISTRY from trigger_sdk.types import TaskConfig, RetryConfig, QueueConfig +from trigger_sdk.ipc import IpcConnection, StdioIpcConnection +from trigger_sdk.schemas.messages import WorkerMessage, CoordinatorMessage +from trigger_sdk.schemas.common import TaskRunExecution __version__ = "0.1.0" __all__ = [ + # Task decorator and registry "task", "Task", "TASK_REGISTRY", + # Configuration types "TaskConfig", "RetryConfig", "QueueConfig", + # IPC layer + "IpcConnection", + "StdioIpcConnection", + # Message types + "WorkerMessage", + "CoordinatorMessage", + # Execution context + "TaskRunExecution", ] diff --git a/packages/python-sdk/trigger_sdk/errors.py b/packages/python-sdk/trigger_sdk/errors.py new file mode 100644 index 0000000000..5f237bec34 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/errors.py @@ -0,0 +1,119 @@ +""" +Exception to TaskRunError conversion utilities. + +Maps Python exceptions to Trigger.dev error schemas for consistent +error reporting across languages. +""" + +import traceback +from typing import Optional + +from trigger_sdk.schemas.errors import ( + TaskRunError, + TaskRunErrorCode, + TaskRunBuiltInError, + TaskRunInternalError, + TaskRunStringError, +) + + +def exception_to_task_run_error(exc: Exception) -> TaskRunError: + """ + Convert Python exception to TaskRunError schema. + + Preserves stack traces and maps exceptions to appropriate error types: + - Built-in Python exceptions → TaskRunBuiltInError + - System/import errors → TaskRunInternalError with error code + - Unknown exceptions → TaskRunStringError (fallback) + + Args: + exc: Python exception to convert + + Returns: + TaskRunError schema object (discriminated union) + """ + # Get stack trace + stack_trace = traceback.format_exc() + + # Try to map to error code + error_code = get_error_code_for_exception(exc) + + # Built-in Python exceptions + if isinstance(exc, ( + TypeError, + ValueError, + AttributeError, + KeyError, + IndexError, + RuntimeError, + AssertionError, + ZeroDivisionError, + NameError, + FileNotFoundError, + PermissionError, + TimeoutError, + )): + return TaskRunBuiltInError( + type="BUILT_IN_ERROR", + name=exc.__class__.__name__, + message=str(exc), + stackTrace=stack_trace, + ) + + # Internal/system errors with error codes + if error_code: + return TaskRunInternalError( + type="INTERNAL_ERROR", + code=error_code, + message=str(exc), + stackTrace=stack_trace, + ) + + # Fallback: string error for unknown exceptions + return TaskRunStringError( + type="STRING_ERROR", + raw=f"{exc.__class__.__name__}: {str(exc)}\n{stack_trace}", + ) + + +def get_error_code_for_exception(exc: Exception) -> Optional[TaskRunErrorCode]: + """ + Map Python exception types to TaskRunErrorCode enum. + + Args: + exc: Python exception + + Returns: + Error code string if mapped, None otherwise + """ + # Import errors + if isinstance(exc, (ImportError, ModuleNotFoundError)): + return "COULD_NOT_IMPORT_TASK" + + # Process exit errors + if isinstance(exc, SystemExit): + return "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE" + + # Cancellation errors + if isinstance(exc, (KeyboardInterrupt, asyncio.CancelledError)): + return "TASK_RUN_CANCELLED" + + # Syntax/parsing errors + if isinstance(exc, (SyntaxError, IndentationError)): + return "TASK_INPUT_ERROR" + + # Timeout errors + if isinstance(exc, TimeoutError): + return "MAX_DURATION_EXCEEDED" + + # Serialization errors (often from JSON encoding/decoding) + if isinstance(exc, (json.JSONDecodeError, UnicodeError)): + return "TASK_OUTPUT_ERROR" + + # Generic fallback + return "TASK_EXECUTION_FAILED" + + +# Import asyncio and json for type checking +import asyncio +import json diff --git a/packages/python-sdk/trigger_sdk/ipc/__init__.py b/packages/python-sdk/trigger_sdk/ipc/__init__.py new file mode 100644 index 0000000000..9a607f72c6 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/ipc/__init__.py @@ -0,0 +1,11 @@ +""" +IPC (Inter-Process Communication) layer for Python workers. + +Provides transport-agnostic message communication between Python workers +and the Node.js coordinator. +""" + +from trigger_sdk.ipc.base import IpcConnection +from trigger_sdk.ipc.stdio import StdioIpcConnection + +__all__ = ["IpcConnection", "StdioIpcConnection"] diff --git a/packages/python-sdk/trigger_sdk/ipc/base.py b/packages/python-sdk/trigger_sdk/ipc/base.py new file mode 100644 index 0000000000..ac0bc7705d --- /dev/null +++ b/packages/python-sdk/trigger_sdk/ipc/base.py @@ -0,0 +1,169 @@ +""" +Abstract IPC connection interface for transport independence. + +Allows multiple transport implementations: +- StdioIpcConnection (line-delimited JSON over stdio) +- GrpcIpcConnection (future: gRPC streaming) +- WebSocketIpcConnection (future: WebSocket transport) +""" + +import json +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, Optional, Union +import traceback + +from trigger_sdk.schemas.messages import ( + WorkerMessage, + CoordinatorMessage, + TaskRunCompletedMessage, + TaskRunFailedMessage, + TaskHeartbeatMessage, +) +from trigger_sdk.schemas.common import ( + TaskRunSuccessfulExecutionResult, + TaskRunFailedExecutionResult, + TaskRunExecutionUsage, +) + + +class IpcConnection(ABC): + """ + Abstract IPC connection interface. + + Implementations must provide transport-specific message sending/receiving + while maintaining the same high-level API for task workers. + """ + + @abstractmethod + async def send(self, message: WorkerMessage) -> None: + """ + Send a message to the coordinator. + + Args: + message: Worker message to send (discriminated union type) + + Raises: + Exception: If sending fails (implementation-specific) + """ + pass + + @abstractmethod + async def start_listening(self) -> None: + """ + Start receiving messages from the coordinator. + + This should be a long-running method that continuously listens for + incoming messages and dispatches them to registered handlers. + + Implementations should handle errors gracefully and not crash on + malformed messages. + """ + pass + + @abstractmethod + def on(self, message_type: str, handler: Callable[[Any], Any]) -> None: + """ + Register a handler for a specific message type. + + Args: + message_type: Message type to handle (e.g., "EXECUTE_TASK_RUN") + handler: Callable that receives the message + Can be sync or async function + """ + pass + + @abstractmethod + def stop(self) -> None: + """ + Stop the connection gracefully. + + Should clean up resources and stop listening for messages. + """ + pass + + # ======================================================================== + # Convenience methods (concrete implementations using abstract methods) + # ======================================================================== + + async def send_completed( + self, + id: str, + output: Any, + usage: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Send TASK_RUN_COMPLETED message. + + Args: + id: Run ID from execution context + output: Task result (will be JSON-serialized if not string) + usage: Optional usage metrics (durationMs, etc.) + """ + # Serialize output to JSON string + if not isinstance(output, str): + output_str = json.dumps(output) + else: + output_str = output + + # Create usage object if provided + usage_obj = None + if usage: + usage_obj = TaskRunExecutionUsage(**usage) + + # Create success result + result = TaskRunSuccessfulExecutionResult( + id=id, + output=output_str, + outputType="application/json", + usage=usage_obj, + ) + + # Create and send message + message = TaskRunCompletedMessage.from_result(result) + await self.send(message) + + async def send_failed( + self, + id: str, + error: Exception, + usage: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Send TASK_RUN_FAILED_TO_RUN message. + + Args: + id: Run ID from execution context + error: Exception that caused the failure + usage: Optional usage metrics + """ + # Import here to avoid circular dependency + from trigger_sdk.errors import exception_to_task_run_error + + # Convert exception to TaskRunError schema + error_obj = exception_to_task_run_error(error) + + # Create usage object if provided + usage_obj = None + if usage: + usage_obj = TaskRunExecutionUsage(**usage) + + # Create failure result + result = TaskRunFailedExecutionResult( + id=id, + error=error_obj, + usage=usage_obj, + ) + + # Create and send message + message = TaskRunFailedMessage.from_result(result) + await self.send(message) + + async def send_heartbeat(self, id: str) -> None: + """ + Send TASK_HEARTBEAT message. + + Args: + id: Run or attempt ID + """ + message = TaskHeartbeatMessage(id=id) + await self.send(message) diff --git a/packages/python-sdk/trigger_sdk/ipc/stdio.py b/packages/python-sdk/trigger_sdk/ipc/stdio.py new file mode 100644 index 0000000000..e788594c2b --- /dev/null +++ b/packages/python-sdk/trigger_sdk/ipc/stdio.py @@ -0,0 +1,134 @@ +""" +Stdio-based IPC implementation using line-delimited JSON. + +Communicates with Node.js coordinator via stdin/stdout: +- Reads messages from stdin (coordinator → worker) +- Writes messages to stdout (worker → coordinator) +- Logs to stderr (won't interfere with IPC) +""" + +import sys +import json +import asyncio +import traceback +from typing import Any, Callable, Dict +from pydantic import ValidationError, TypeAdapter + +from trigger_sdk.ipc.base import IpcConnection +from trigger_sdk.schemas.messages import WorkerMessage, CoordinatorMessage + +# Create type adapters for union types +_coordinator_message_adapter: TypeAdapter[CoordinatorMessage] = TypeAdapter(CoordinatorMessage) + + +class StdioIpcConnection(IpcConnection): + """ + Stdio-based IPC using line-delimited JSON. + + Compatible with Node.js child_process.spawn() stdio communication. + Thread-safe message sending with async locks. + """ + + def __init__(self) -> None: + self._handlers: Dict[str, Callable[[Any], Any]] = {} + self._running = False + self._stdout_lock = asyncio.Lock() + + def on(self, message_type: str, handler: Callable[[Any], Any]) -> None: + """Register a message handler""" + self._handlers[message_type] = handler + + async def send(self, message: WorkerMessage) -> None: + """ + Send message to stdout as line-delimited JSON. + + Thread-safe using asyncio.Lock to prevent message interleaving. + Errors are logged to stderr (won't interfere with IPC). + """ + try: + # Serialize message to JSON + json_str = message.model_dump_json() + + # Write to stdout with lock (prevent interleaving) + async with self._stdout_lock: + sys.stdout.write(json_str + "\n") + sys.stdout.flush() + + except Exception as e: + # Log to stderr (won't interfere with IPC) + sys.stderr.write(f"[IPC] Failed to send message: {e}\n") + sys.stderr.flush() + + async def start_listening(self) -> None: + """ + Read line-delimited JSON from stdin. + + Continuously reads messages and dispatches to registered handlers. + Handles errors gracefully without crashing: + - Malformed JSON → logged to stderr, continues + - Validation errors → logged to stderr, continues + - Handler exceptions → logged to stderr, continues + """ + self._running = True + loop = asyncio.get_event_loop() + + try: + while self._running: + # Non-blocking readline using executor + line = await loop.run_in_executor(None, sys.stdin.readline) + + # Check for EOF + if not line: + break + + line = line.strip() + if not line: + continue + + try: + # Parse JSON + data = json.loads(line) + + # Get message type + message_type = data.get("type") + if not message_type: + sys.stderr.write(f"[IPC] Message missing 'type' field: {line}\n") + sys.stderr.flush() + continue + + # Validate message schema with Pydantic + message = _coordinator_message_adapter.validate_python(data) + + # Dispatch to registered handler + if message_type in self._handlers: + handler = self._handlers[message_type] + + # Support both sync and async handlers + if asyncio.iscoroutinefunction(handler): + await handler(message) + else: + handler(message) + else: + sys.stderr.write(f"[IPC] No handler for message type: {message_type}\n") + sys.stderr.flush() + + except json.JSONDecodeError as e: + sys.stderr.write(f"[IPC] Invalid JSON: {line}\n") + sys.stderr.write(f"[IPC] Error: {e}\n") + sys.stderr.flush() + + except ValidationError as e: + sys.stderr.write(f"[IPC] Message validation failed: {e}\n") + sys.stderr.flush() + + except Exception as e: + sys.stderr.write(f"[IPC] Handler error: {e}\n") + sys.stderr.write(f"[IPC] Traceback: {traceback.format_exc()}\n") + sys.stderr.flush() + + finally: + self._running = False + + def stop(self) -> None: + """Stop listening for messages""" + self._running = False diff --git a/packages/python-sdk/trigger_sdk/schemas/__init__.py b/packages/python-sdk/trigger_sdk/schemas/__init__.py new file mode 100644 index 0000000000..d36102cb37 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/schemas/__init__.py @@ -0,0 +1,86 @@ +""" +Pydantic message schemas for Python SDK. + +Matches TypeScript Zod schemas in packages/core/src/v3/schemas/ +""" + +from trigger_sdk.schemas.common import ( + TaskRunExecutionUsage, + TaskRunExecutionRetry, + TaskInfo, + RunInfo, + AttemptInfo, + OrganizationInfo, + ProjectInfo, + EnvironmentInfo, + QueueInfo, + DeploymentInfo, + TaskRunExecution, + TaskRunSuccessfulExecutionResult, + TaskRunFailedExecutionResult, + TaskRunExecutionResult, +) + +from trigger_sdk.schemas.errors import ( + TaskRunErrorCode, + TaskRunBuiltInError, + TaskRunInternalError, + TaskRunStringError, + TaskRunError, +) + +from trigger_sdk.schemas.messages import ( + TaskRunCompletedMessage, + TaskRunFailedMessage, + TaskHeartbeatMessage, + IndexTasksCompleteMessage, + WorkerMessage, + ExecuteTaskRunMessage, + CancelMessage, + FlushMessage, + CoordinatorMessage, +) + +from trigger_sdk.schemas.resources import ( + QueueConfig, + RetryConfig, + TaskResource, +) + +__all__ = [ + # Common + "TaskRunExecutionUsage", + "TaskRunExecutionRetry", + "TaskInfo", + "RunInfo", + "AttemptInfo", + "OrganizationInfo", + "ProjectInfo", + "EnvironmentInfo", + "QueueInfo", + "DeploymentInfo", + "TaskRunExecution", + "TaskRunSuccessfulExecutionResult", + "TaskRunFailedExecutionResult", + "TaskRunExecutionResult", + # Errors + "TaskRunErrorCode", + "TaskRunBuiltInError", + "TaskRunInternalError", + "TaskRunStringError", + "TaskRunError", + # Messages + "TaskRunCompletedMessage", + "TaskRunFailedMessage", + "TaskHeartbeatMessage", + "IndexTasksCompleteMessage", + "WorkerMessage", + "ExecuteTaskRunMessage", + "CancelMessage", + "FlushMessage", + "CoordinatorMessage", + # Resources + "QueueConfig", + "RetryConfig", + "TaskResource", +] diff --git a/packages/python-sdk/trigger_sdk/schemas/common.py b/packages/python-sdk/trigger_sdk/schemas/common.py new file mode 100644 index 0000000000..2e0adebfa6 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/schemas/common.py @@ -0,0 +1,138 @@ +""" +Core execution types and task run result schemas. + +These schemas match TypeScript types in: +packages/core/src/v3/schemas/common.ts +""" + +from typing import Any, Dict, Literal, Optional +from pydantic import BaseModel, Field + +from trigger_sdk.schemas.errors import TaskRunError + + +class TaskRunExecutionUsage(BaseModel): + """Task execution usage metrics""" + durationMs: int + + +class TaskRunExecutionRetry(BaseModel): + """Task retry information""" + timestamp: int # Unix timestamp + delay: int # Delay in milliseconds + + +class TaskInfo(BaseModel): + """Basic task information""" + id: str + filePath: str + + +class RunInfo(BaseModel): + """Task run information""" + id: str + payload: str # JSON-serialized payload + payloadType: str + tags: list[str] = Field(default_factory=list) + isTest: bool = False + + +class AttemptInfo(BaseModel): + """Task attempt information""" + id: str + number: int + startedAt: str # ISO 8601 timestamp + + +# Progressive expansion fields - to be implemented later +class OrganizationInfo(BaseModel): + """Organization context (TODO: expand)""" + id: str + slug: str + name: str + + +class ProjectInfo(BaseModel): + """Project context (TODO: expand)""" + id: str + ref: str + slug: str + name: str + + +class EnvironmentInfo(BaseModel): + """Environment context (TODO: expand)""" + id: str + slug: str + type: Literal["PRODUCTION", "STAGING", "DEVELOPMENT", "PREVIEW"] + + +class QueueInfo(BaseModel): + """Queue context (TODO: expand)""" + id: str + name: str + + +class DeploymentInfo(BaseModel): + """Deployment context (TODO: expand)""" + id: str + shortCode: str + version: str + + +class TaskRunExecution(BaseModel): + """ + Complete task execution context. + + Progressive design: Essential fields required, optional fields for future expansion. + Maps to TypeScript TaskRunExecution schema. + """ + # Essential fields (MVP) + task: TaskInfo + run: RunInfo + attempt: AttemptInfo + + # Optional fields for progressive expansion + # TODO: Make these required once coordinator integration is complete + queue: Optional[QueueInfo] = None + organization: Optional[OrganizationInfo] = None + project: Optional[ProjectInfo] = None + environment: Optional[EnvironmentInfo] = None + deployment: Optional[DeploymentInfo] = None + + +class TaskRunSuccessfulExecutionResult(BaseModel): + """ + Successful task execution result. + + Maps to TypeScript TaskRunSuccessfulExecutionResult. + Returned when task completes successfully. + """ + ok: Literal[True] = True + id: str # Run ID from execution context + output: Optional[str] = None # JSON-serialized output + outputType: str = "application/json" + usage: Optional[TaskRunExecutionUsage] = None + taskIdentifier: Optional[str] = None # For backwards compatibility + # Note: Skipping metadata/flushedMetadata for MVP + + +class TaskRunFailedExecutionResult(BaseModel): + """ + Failed task execution result. + + Maps to TypeScript TaskRunFailedExecutionResult. + Returned when task fails with an error. + """ + ok: Literal[False] = False + id: str # Run ID from execution context + error: TaskRunError + retry: Optional[TaskRunExecutionRetry] = None + skippedRetrying: Optional[bool] = None + usage: Optional[TaskRunExecutionUsage] = None + taskIdentifier: Optional[str] = None # For backwards compatibility + # Note: Skipping metadata/flushedMetadata for MVP + + +# Type alias for result discriminated union +TaskRunExecutionResult = TaskRunSuccessfulExecutionResult | TaskRunFailedExecutionResult diff --git a/packages/python-sdk/trigger_sdk/schemas/errors.py b/packages/python-sdk/trigger_sdk/schemas/errors.py new file mode 100644 index 0000000000..510a6c5a9a --- /dev/null +++ b/packages/python-sdk/trigger_sdk/schemas/errors.py @@ -0,0 +1,68 @@ +""" +TaskRunError types and error code mappings. + +These schemas match the TypeScript error types in: +packages/core/src/v3/schemas/common.ts (lines 130-207) +""" + +from typing import Literal, Union +from pydantic import BaseModel + + +# Essential subset of 38 error codes - focused on Python worker lifecycle +TaskRunErrorCode = Literal[ + "COULD_NOT_IMPORT_TASK", + "TASK_EXECUTION_FAILED", + "TASK_RUN_CANCELLED", + "MAX_DURATION_EXCEEDED", + "TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE", + "TASK_INPUT_ERROR", + "TASK_OUTPUT_ERROR", + "INTERNAL_ERROR", +] + + +class TaskRunBuiltInError(BaseModel): + """ + Built-in Python exception error. + + Used for standard Python exceptions like TypeError, ValueError, etc. + Maps to TypeScript BUILT_IN_ERROR type. + """ + type: Literal["BUILT_IN_ERROR"] = "BUILT_IN_ERROR" + name: str # Exception class name (e.g., "TypeError", "ValueError") + message: str + stackTrace: str + + +class TaskRunInternalError(BaseModel): + """ + Internal system error with specific error code. + + Used for system-level errors during task execution. + Maps to TypeScript INTERNAL_ERROR type. + """ + type: Literal["INTERNAL_ERROR"] = "INTERNAL_ERROR" + code: TaskRunErrorCode + message: str = "" + stackTrace: str = "" + + +class TaskRunStringError(BaseModel): + """ + Simple string error. + + Used as fallback for errors that don't fit other categories. + Maps to TypeScript STRING_ERROR type. + """ + type: Literal["STRING_ERROR"] = "STRING_ERROR" + raw: str + + +# Discriminated union of all error types +# Note: Skipping CUSTOM_ERROR for MVP - can be added later +TaskRunError = Union[ + TaskRunBuiltInError, + TaskRunInternalError, + TaskRunStringError, +] diff --git a/packages/python-sdk/trigger_sdk/schemas/messages.py b/packages/python-sdk/trigger_sdk/schemas/messages.py new file mode 100644 index 0000000000..577378922a --- /dev/null +++ b/packages/python-sdk/trigger_sdk/schemas/messages.py @@ -0,0 +1,130 @@ +""" +IPC message schemas for worker-coordinator communication. + +These schemas match TypeScript message types in: +packages/core/src/v3/schemas/messages.ts +""" + +from typing import Any, Dict, Literal, Union +from pydantic import BaseModel, Field + +from trigger_sdk.schemas.common import ( + TaskRunSuccessfulExecutionResult, + TaskRunFailedExecutionResult, + TaskRunExecution, +) + + +# ============================================================================ +# Worker → Coordinator Messages +# ============================================================================ + +class TaskRunCompletedMessage(BaseModel): + """ + Task execution completed successfully. + + Sent when a task finishes execution without errors. + """ + type: Literal["TASK_RUN_COMPLETED"] = "TASK_RUN_COMPLETED" + version: Literal["v1"] = "v1" + completion: Dict[str, Any] # TaskRunSuccessfulExecutionResult as dict + + @classmethod + def from_result(cls, result: TaskRunSuccessfulExecutionResult) -> "TaskRunCompletedMessage": + """Create message from result object""" + return cls(completion=result.model_dump()) + + +class TaskRunFailedMessage(BaseModel): + """ + Task execution failed. + + Sent when a task fails with an error. + """ + type: Literal["TASK_RUN_FAILED_TO_RUN"] = "TASK_RUN_FAILED_TO_RUN" + version: Literal["v1"] = "v1" + completion: Dict[str, Any] # TaskRunFailedExecutionResult as dict + + @classmethod + def from_result(cls, result: TaskRunFailedExecutionResult) -> "TaskRunFailedMessage": + """Create message from result object""" + return cls(completion=result.model_dump()) + + +class TaskHeartbeatMessage(BaseModel): + """ + Heartbeat indicating task is still running. + + Sent periodically during long-running task execution. + """ + type: Literal["TASK_HEARTBEAT"] = "TASK_HEARTBEAT" + version: Literal["v1"] = "v1" + id: str # Run or attempt ID + + +class IndexTasksCompleteMessage(BaseModel): + """ + Task indexing completed. + + Sent after discovering and indexing all tasks in the project. + Contains task catalog with metadata. + """ + type: Literal["INDEX_TASKS_COMPLETE"] = "INDEX_TASKS_COMPLETE" + version: Literal["v1"] = "v1" + tasks: list[Dict[str, Any]] # List of TaskResource as dicts + + +# Discriminated union of all worker messages +WorkerMessage = Union[ + TaskRunCompletedMessage, + TaskRunFailedMessage, + TaskHeartbeatMessage, + IndexTasksCompleteMessage, +] + + +# ============================================================================ +# Coordinator → Worker Messages +# ============================================================================ + +class ExecuteTaskRunMessage(BaseModel): + """ + Execute a task run. + + Coordinator sends this to worker to start task execution. + """ + type: Literal["EXECUTE_TASK_RUN"] = "EXECUTE_TASK_RUN" + version: Literal["v1"] = "v1" + execution: Dict[str, Any] # TaskRunExecution as dict + + def get_execution(self) -> TaskRunExecution: + """Parse execution payload""" + return TaskRunExecution.model_validate(self.execution) + + +class CancelMessage(BaseModel): + """ + Cancel current task run. + + Coordinator sends this to gracefully stop task execution. + """ + type: Literal["CANCEL"] = "CANCEL" + version: Literal["v1"] = "v1" + + +class FlushMessage(BaseModel): + """ + Flush logs and telemetry. + + Coordinator sends this to ensure logs are sent before shutdown. + """ + type: Literal["FLUSH"] = "FLUSH" + version: Literal["v1"] = "v1" + + +# Discriminated union of all coordinator messages +CoordinatorMessage = Union[ + ExecuteTaskRunMessage, + CancelMessage, + FlushMessage, +] diff --git a/packages/python-sdk/trigger_sdk/schemas/resources.py b/packages/python-sdk/trigger_sdk/schemas/resources.py new file mode 100644 index 0000000000..8fe9fe86a5 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/schemas/resources.py @@ -0,0 +1,46 @@ +""" +Task resource schemas for indexing and metadata. + +These schemas match TypeScript TaskResource in: +packages/core/src/v3/schemas/resources.ts +""" + +from typing import Optional +from pydantic import BaseModel + + +class QueueConfig(BaseModel): + """Queue configuration for a task""" + name: Optional[str] = None + concurrencyLimit: Optional[int] = None + + +class RetryConfig(BaseModel): + """Retry configuration for a task""" + maxAttempts: Optional[int] = None + factor: Optional[float] = None + minTimeoutInMs: Optional[int] = None + maxTimeoutInMs: Optional[int] = None + randomize: Optional[bool] = None + + +class TaskResource(BaseModel): + """ + Task metadata for indexing. + + Sent to coordinator during task discovery/indexing phase. + Maps to TypeScript TaskResource schema. + """ + id: str + filePath: str + exportName: str # Python uses task ID as export name + description: Optional[str] = None + queue: Optional[QueueConfig] = None + retry: Optional[RetryConfig] = None + maxDuration: Optional[int] = None # In seconds + + # TODO: Add in future iterations + # machine: Optional[MachineConfig] = None + # triggerSource: Optional[str] = None + # schedule: Optional[ScheduleMetadata] = None + # payloadSchema: Optional[dict] = None From cfa3a188e73db5d75624a39a8aa0ff738159c545 Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sun, 9 Nov 2025 00:35:39 +0300 Subject: [PATCH 03/10] feat(python-sdk): implement task execution context and structured logging --- .gitignore | 12 +- packages/python-sdk/tests/test_context.py | 169 ++++++++++++++++++ packages/python-sdk/tests/test_logger.py | 147 +++++++++++++++ packages/python-sdk/trigger_sdk/__init__.py | 6 + packages/python-sdk/trigger_sdk/context.py | 78 ++++++++ packages/python-sdk/trigger_sdk/logger.py | 68 +++++++ .../trigger_sdk/schemas/__init__.py | 2 + .../python-sdk/trigger_sdk/schemas/common.py | 6 + packages/python-sdk/trigger_sdk/telemetry.py | 66 +++++++ 9 files changed, 553 insertions(+), 1 deletion(-) create mode 100644 packages/python-sdk/tests/test_context.py create mode 100644 packages/python-sdk/tests/test_logger.py create mode 100644 packages/python-sdk/trigger_sdk/context.py create mode 100644 packages/python-sdk/trigger_sdk/logger.py create mode 100644 packages/python-sdk/trigger_sdk/telemetry.py diff --git a/.gitignore b/.gitignore index 6f435d0400..fa4cd81734 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,14 @@ apps/**/public/build /packages/trigger-sdk/src/package.json /packages/python/src/package.json .claude -.mcp.log \ No newline at end of file +.mcp.log + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +.mypy_cache/ +.pytest_cache/ +.venv/ +build/ \ No newline at end of file diff --git a/packages/python-sdk/tests/test_context.py b/packages/python-sdk/tests/test_context.py new file mode 100644 index 0000000000..c618a25e01 --- /dev/null +++ b/packages/python-sdk/tests/test_context.py @@ -0,0 +1,169 @@ +"""Tests for task context""" + +import pytest +from trigger_sdk.context import ( + TaskContext, + get_current_context, + set_current_context, + clear_current_context, +) +from trigger_sdk.schemas.common import ( + TaskInfo, + RunInfo, + AttemptInfo, + BatchInfo, + TaskRunExecution, + EnvironmentInfo, +) + + +def test_create_context(): + """Test creating task context""" + context = TaskContext( + task=TaskInfo(id="test-task", filePath="/test.py"), + run=RunInfo( + id="run_123", + payload='{"value": 42}', + payloadType="application/json", + tags=[], + isTest=False, + ), + attempt=AttemptInfo( + id="attempt_123", + number=1, + startedAt="2024-01-01T00:00:00Z", + ), + ) + + assert context.task.id == "test-task" + assert context.run.id == "run_123" + assert context.attempt.number == 1 + assert context.is_retry is False + + +def test_is_retry(): + """Test retry detection""" + context = TaskContext( + task=TaskInfo(id="test", filePath="/test.py"), + run=RunInfo( + id="r1", + payload="{}", + payloadType="json", + tags=[], + isTest=False, + ), + attempt=AttemptInfo( + id="a1", + number=3, + startedAt="2024-01-01T00:00:00Z", + ), + ) + + assert context.is_retry is True + + +def test_context_var(): + """Test context variable get/set""" + assert get_current_context() is None + + context = TaskContext( + task=TaskInfo(id="test", filePath="/test.py"), + run=RunInfo( + id="r1", + payload="{}", + payloadType="json", + tags=[], + isTest=False, + ), + attempt=AttemptInfo( + id="a1", + number=1, + startedAt="", + ), + ) + + set_current_context(context) + assert get_current_context() == context + + clear_current_context() + assert get_current_context() is None + + +def test_context_from_execution_payload(): + """Test creating context from TaskRunExecution""" + execution = TaskRunExecution( + task=TaskInfo(id="test", filePath="/test.py"), + run=RunInfo( + id="run_123", + payload='{"value": 42}', + payloadType="application/json", + tags=[], + isTest=False, + ), + attempt=AttemptInfo( + id="attempt_123", + number=1, + startedAt="2024-01-01T00:00:00Z", + ), + batch=BatchInfo(id="batch_123"), + environment=EnvironmentInfo( + id="env_123", + slug="prod", + type="PRODUCTION", + ), + ) + + context = TaskContext.from_execution_payload(execution) + + assert context.task.id == "test" + assert context.run.id == "run_123" + assert context.attempt.id == "attempt_123" + assert context.batch is not None + assert context.batch.id == "batch_123" + assert context.environment["slug"] == "prod" + assert context.environment["type"] == "PRODUCTION" + + +def test_context_from_execution_without_optional_fields(): + """Test creating context from minimal TaskRunExecution""" + execution = TaskRunExecution( + task=TaskInfo(id="test", filePath="/test.py"), + run=RunInfo( + id="run_123", + payload='{"value": 42}', + payloadType="application/json", + ), + attempt=AttemptInfo( + id="attempt_123", + number=1, + startedAt="2024-01-01T00:00:00Z", + ), + ) + + context = TaskContext.from_execution_payload(execution) + + assert context.task.id == "test" + assert context.batch is None + assert context.environment == {} + + +def test_context_repr(): + """Test context string representation""" + context = TaskContext( + task=TaskInfo(id="my-task", filePath="/test.py"), + run=RunInfo( + id="run_abc", + payload="{}", + payloadType="json", + ), + attempt=AttemptInfo( + id="attempt_xyz", + number=2, + startedAt="", + ), + ) + + repr_str = repr(context) + assert "my-task" in repr_str + assert "run_abc" in repr_str + assert "2" in repr_str diff --git a/packages/python-sdk/tests/test_logger.py b/packages/python-sdk/tests/test_logger.py new file mode 100644 index 0000000000..5fd4b1f66f --- /dev/null +++ b/packages/python-sdk/tests/test_logger.py @@ -0,0 +1,147 @@ +"""Tests for structured logging""" + +import pytest +import json +from io import StringIO +from unittest.mock import patch +from trigger_sdk.logger import TaskLogger +from trigger_sdk.context import ( + TaskContext, + set_current_context, + clear_current_context, +) +from trigger_sdk.schemas.common import TaskInfo, RunInfo, AttemptInfo + + +def test_logger_basic(): + """Test basic logging""" + logger = TaskLogger("test") + + with patch("sys.stderr", new=StringIO()) as mock_stderr: + logger.info("Test message", extra_field="value") + + output = mock_stderr.getvalue() + log_data = json.loads(output.strip()) + + assert log_data["level"] == "INFO" + assert log_data["message"] == "Test message" + assert log_data["extra_field"] == "value" + assert log_data["logger"] == "test" + assert "timestamp" in log_data + + +def test_logger_levels(): + """Test different log levels""" + logger = TaskLogger("test") + + levels = ["DEBUG", "INFO", "WARN", "ERROR"] + + for level in levels: + with patch("sys.stderr", new=StringIO()) as mock_stderr: + method = getattr(logger, level.lower()) + method(f"{level} message") + + output = mock_stderr.getvalue() + log_data = json.loads(output.strip()) + + assert log_data["level"] == level + assert log_data["message"] == f"{level} message" + + +def test_logger_custom_level(): + """Test custom log level""" + logger = TaskLogger("test") + + with patch("sys.stderr", new=StringIO()) as mock_stderr: + logger.log("CUSTOM", "Custom message") + + output = mock_stderr.getvalue() + log_data = json.loads(output.strip()) + + assert log_data["level"] == "CUSTOM" + assert log_data["message"] == "Custom message" + + +def test_logger_with_context(): + """Test logging with task context""" + context = TaskContext( + task=TaskInfo(id="test-task", filePath="/test.py"), + run=RunInfo( + id="run_123", + payload="{}", + payloadType="json", + tags=[], + isTest=False, + ), + attempt=AttemptInfo( + id="attempt_123", + number=2, + startedAt="", + ), + ) + set_current_context(context) + + logger = TaskLogger("test") + + with patch("sys.stderr", new=StringIO()) as mock_stderr: + logger.info("With context") + + output = mock_stderr.getvalue() + log_data = json.loads(output.strip()) + + assert log_data["task"]["id"] == "test-task" + assert log_data["task"]["runId"] == "run_123" + assert log_data["task"]["attemptId"] == "attempt_123" + assert log_data["task"]["attemptNumber"] == 2 + + clear_current_context() + + +def test_logger_without_context(): + """Test logging without task context""" + clear_current_context() # Ensure no context + logger = TaskLogger("test") + + with patch("sys.stderr", new=StringIO()) as mock_stderr: + logger.info("Without context") + + output = mock_stderr.getvalue() + log_data = json.loads(output.strip()) + + assert "task" not in log_data + assert log_data["message"] == "Without context" + + +def test_logger_json_format(): + """Test JSON output format validation""" + logger = TaskLogger("test") + + with patch("sys.stderr", new=StringIO()) as mock_stderr: + logger.info("Test", field1="value1", field2=42, field3=True) + + output = mock_stderr.getvalue() + log_data = json.loads(output.strip()) + + # Validate all expected fields + assert "timestamp" in log_data + assert "level" in log_data + assert "message" in log_data + assert "logger" in log_data + assert log_data["field1"] == "value1" + assert log_data["field2"] == 42 + assert log_data["field3"] is True + + +def test_logger_timestamp_format(): + """Test timestamp format is ISO 8601 with Z suffix""" + logger = TaskLogger("test") + + with patch("sys.stderr", new=StringIO()) as mock_stderr: + logger.info("Test timestamp") + + output = mock_stderr.getvalue() + log_data = json.loads(output.strip()) + + timestamp = log_data["timestamp"] + assert timestamp.endswith("Z") + assert "T" in timestamp # ISO 8601 format diff --git a/packages/python-sdk/trigger_sdk/__init__.py b/packages/python-sdk/trigger_sdk/__init__.py index 015341a723..e9a61230ce 100644 --- a/packages/python-sdk/trigger_sdk/__init__.py +++ b/packages/python-sdk/trigger_sdk/__init__.py @@ -5,6 +5,8 @@ from trigger_sdk.ipc import IpcConnection, StdioIpcConnection from trigger_sdk.schemas.messages import WorkerMessage, CoordinatorMessage from trigger_sdk.schemas.common import TaskRunExecution +from trigger_sdk.context import TaskContext, get_current_context +from trigger_sdk.logger import logger __version__ = "0.1.0" __all__ = [ @@ -24,4 +26,8 @@ "CoordinatorMessage", # Execution context "TaskRunExecution", + "TaskContext", + "get_current_context", + # Logging + "logger", ] diff --git a/packages/python-sdk/trigger_sdk/context.py b/packages/python-sdk/trigger_sdk/context.py new file mode 100644 index 0000000000..54a0ca89b2 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/context.py @@ -0,0 +1,78 @@ +"""Task execution context""" + +from typing import Any, Dict, Optional +from contextvars import ContextVar + +from trigger_sdk.schemas.common import ( + TaskInfo, + RunInfo, + AttemptInfo, + BatchInfo, + TaskRunExecution, +) + +# Context variable for current task execution +_current_context: ContextVar[Optional["TaskContext"]] = ContextVar( + "current_task_context", default=None +) + + +class TaskContext: + """ + Context object available during task execution. + + Provides access to task metadata, run info, and utilities. + """ + + def __init__( + self, + task: TaskInfo, + run: RunInfo, + attempt: AttemptInfo, + batch: Optional[BatchInfo] = None, + environment: Optional[Dict[str, Any]] = None, + ): + self.task = task + self.run = run + self.attempt = attempt + self.batch = batch + self.environment = environment or {} + + @classmethod + def from_execution_payload(cls, execution: TaskRunExecution) -> "TaskContext": + """Create context from execution message payload""" + # Convert EnvironmentInfo to dict for MVP + env_dict = {} + if execution.environment: + env_dict = execution.environment.model_dump() + + return cls( + task=execution.task, + run=execution.run, + attempt=execution.attempt, + batch=execution.batch, + environment=env_dict, + ) + + @property + def is_retry(self) -> bool: + """Check if this is a retry attempt""" + return self.attempt.number > 1 + + def __repr__(self) -> str: + return f"TaskContext(task={self.task.id}, run={self.run.id}, attempt={self.attempt.number})" + + +def get_current_context() -> Optional[TaskContext]: + """Get the current task context (if inside a task execution)""" + return _current_context.get() + + +def set_current_context(context: TaskContext) -> None: + """Set the current task context (called by worker)""" + _current_context.set(context) + + +def clear_current_context() -> None: + """Clear the current task context""" + _current_context.set(None) diff --git a/packages/python-sdk/trigger_sdk/logger.py b/packages/python-sdk/trigger_sdk/logger.py new file mode 100644 index 0000000000..707cb81d16 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/logger.py @@ -0,0 +1,68 @@ +"""Structured logging for tasks""" + +import json +import sys +from typing import Any +from datetime import datetime, timezone + +from trigger_sdk.context import get_current_context + + +class TaskLogger: + """ + Structured logger for task execution. + + Logs are sent to stderr with structured metadata. + """ + + def __init__(self, name: str = "trigger"): + self.name = name + + def _log(self, level: str, message: str, **extra: Any) -> None: + """Internal log method with structured data""" + context = get_current_context() + + log_data = { + "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "level": level, + "message": message, + "logger": self.name, + **extra, + } + + # Add context metadata if available + if context: + log_data["task"] = { + "id": context.task.id, + "runId": context.run.id, + "attemptId": context.attempt.id, + "attemptNumber": context.attempt.number, + } + + # Write to stderr as JSON + sys.stderr.write(json.dumps(log_data) + "\n") + sys.stderr.flush() + + def debug(self, message: str, **extra: Any) -> None: + """Log debug message""" + self._log("DEBUG", message, **extra) + + def info(self, message: str, **extra: Any) -> None: + """Log info message""" + self._log("INFO", message, **extra) + + def warn(self, message: str, **extra: Any) -> None: + """Log warning message""" + self._log("WARN", message, **extra) + + def error(self, message: str, **extra: Any) -> None: + """Log error message""" + self._log("ERROR", message, **extra) + + def log(self, level: str, message: str, **extra: Any) -> None: + """Log with custom level""" + self._log(level.upper(), message, **extra) + + +# Global logger instance +logger = TaskLogger("trigger") diff --git a/packages/python-sdk/trigger_sdk/schemas/__init__.py b/packages/python-sdk/trigger_sdk/schemas/__init__.py index d36102cb37..5463cc12fa 100644 --- a/packages/python-sdk/trigger_sdk/schemas/__init__.py +++ b/packages/python-sdk/trigger_sdk/schemas/__init__.py @@ -10,6 +10,7 @@ TaskInfo, RunInfo, AttemptInfo, + BatchInfo, OrganizationInfo, ProjectInfo, EnvironmentInfo, @@ -54,6 +55,7 @@ "TaskInfo", "RunInfo", "AttemptInfo", + "BatchInfo", "OrganizationInfo", "ProjectInfo", "EnvironmentInfo", diff --git a/packages/python-sdk/trigger_sdk/schemas/common.py b/packages/python-sdk/trigger_sdk/schemas/common.py index 2e0adebfa6..6c928b6d44 100644 --- a/packages/python-sdk/trigger_sdk/schemas/common.py +++ b/packages/python-sdk/trigger_sdk/schemas/common.py @@ -80,6 +80,11 @@ class DeploymentInfo(BaseModel): version: str +class BatchInfo(BaseModel): + """Batch execution context""" + id: str + + class TaskRunExecution(BaseModel): """ Complete task execution context. @@ -94,6 +99,7 @@ class TaskRunExecution(BaseModel): # Optional fields for progressive expansion # TODO: Make these required once coordinator integration is complete + batch: Optional[BatchInfo] = None queue: Optional[QueueInfo] = None organization: Optional[OrganizationInfo] = None project: Optional[ProjectInfo] = None diff --git a/packages/python-sdk/trigger_sdk/telemetry.py b/packages/python-sdk/trigger_sdk/telemetry.py new file mode 100644 index 0000000000..811aee9814 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/telemetry.py @@ -0,0 +1,66 @@ +"""OpenTelemetry integration (minimal implementation)""" + +from typing import Dict, Optional +import os + + +class TraceContext: + """ + Minimal OpenTelemetry trace context. + + Full OpenTelemetry integration can be added later. + """ + + def __init__( + self, + trace_id: Optional[str] = None, + span_id: Optional[str] = None, + trace_flags: Optional[str] = None, + ): + self.trace_id = trace_id + self.span_id = span_id + self.trace_flags = trace_flags + + @classmethod + def from_traceparent(cls, traceparent: str) -> "TraceContext": + """ + Parse W3C traceparent header. + + Format: 00-{trace_id}-{span_id}-{flags} + """ + parts = traceparent.split("-") + if len(parts) != 4: + raise ValueError(f"Invalid traceparent format: {traceparent}") + + return cls( + trace_id=parts[1], + span_id=parts[2], + trace_flags=parts[3], + ) + + @classmethod + def from_env(cls) -> Optional["TraceContext"]: + """Get trace context from TRACEPARENT environment variable""" + traceparent = os.getenv("TRACEPARENT") + if not traceparent: + return None + + return cls.from_traceparent(traceparent) + + def to_traceparent(self) -> str: + """Convert to W3C traceparent header""" + return f"00-{self.trace_id}-{self.span_id}-{self.trace_flags}" + + def inject_env(self) -> Dict[str, str]: + """Get environment variables for propagation""" + if not self.trace_id: + return {} + + return { + "TRACEPARENT": self.to_traceparent(), + } + + +def get_trace_context() -> Optional[TraceContext]: + """Get current trace context from environment""" + return TraceContext.from_env() From 29167897e57ccafd07d738af2ab150d349dd2f45 Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sun, 9 Nov 2025 00:36:46 +0300 Subject: [PATCH 04/10] feat(cli): add Python worker entry points for task execution --- .../cli-v3/src/entryPoints/python/__init__.py | 2 + .../python/managed-index-worker.py | 147 +++++++++++ .../entryPoints/python/managed-run-worker.py | 233 ++++++++++++++++++ packages/cli-v3/tests/python/test-task.py | 30 +++ packages/cli-v3/tests/python/test-workers.sh | 70 ++++++ 5 files changed, 482 insertions(+) create mode 100644 packages/cli-v3/src/entryPoints/python/__init__.py create mode 100755 packages/cli-v3/src/entryPoints/python/managed-index-worker.py create mode 100755 packages/cli-v3/src/entryPoints/python/managed-run-worker.py create mode 100644 packages/cli-v3/tests/python/test-task.py create mode 100755 packages/cli-v3/tests/python/test-workers.sh diff --git a/packages/cli-v3/src/entryPoints/python/__init__.py b/packages/cli-v3/src/entryPoints/python/__init__.py new file mode 100644 index 0000000000..8ee68d2eca --- /dev/null +++ b/packages/cli-v3/src/entryPoints/python/__init__.py @@ -0,0 +1,2 @@ +"""Python worker entry points""" +# Empty init file to make this a Python package diff --git a/packages/cli-v3/src/entryPoints/python/managed-index-worker.py b/packages/cli-v3/src/entryPoints/python/managed-index-worker.py new file mode 100755 index 0000000000..f6c0cd39ce --- /dev/null +++ b/packages/cli-v3/src/entryPoints/python/managed-index-worker.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +""" +Python Index Worker + +Discovers and indexes Python tasks by importing task files. + +Flow: +1. Read BuildManifest from environment or file +2. Import all Python task files +3. Collect tasks from TASK_REGISTRY +4. Send INDEX_TASKS_COMPLETE message with task metadata +""" + +import sys +import json +import os +import asyncio +import importlib.util +import traceback +from pathlib import Path +from typing import Dict, List, Any + +# Import SDK (assumes it's installed via pip) +from trigger_sdk.task import TASK_REGISTRY +from trigger_sdk.ipc import StdioIpcConnection +from trigger_sdk.schemas import IndexTasksCompleteMessage, TaskResource +from trigger_sdk.logger import logger + + +def load_manifest() -> Dict[str, Any]: + """Load build manifest from file or environment""" + manifest_path = os.getenv("TRIGGER_MANIFEST_PATH", "./build-manifest.json") + + try: + with open(manifest_path, "r") as f: + return json.load(f) + except FileNotFoundError: + logger.error(f"Manifest not found at {manifest_path}") + sys.exit(1) + except json.JSONDecodeError as e: + logger.error(f"Invalid manifest JSON: {e}") + sys.exit(1) + + +def import_task_file(file_path: str) -> bool: + """ + Import a Python task file. + + Returns True if successful, False otherwise. + """ + try: + # Resolve absolute path + abs_path = Path(file_path).resolve() + + if not abs_path.exists(): + logger.error(f"Task file not found: {file_path}") + return False + + # Create module name from file path + module_name = abs_path.stem.replace(".", "_") + + # Import module + spec = importlib.util.spec_from_file_location(module_name, abs_path) + if spec is None or spec.loader is None: + logger.error(f"Failed to create module spec for {file_path}") + return False + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + + logger.debug(f"Successfully imported {file_path}") + return True + + except Exception as e: + logger.error(f"Failed to import {file_path}: {e}", exception=traceback.format_exc()) + return False + + +def collect_task_metadata() -> List[Dict[str, Any]]: + """Collect metadata from all registered tasks""" + tasks = [] + + for task_id, task in TASK_REGISTRY.items(): + try: + # Get task metadata + task_meta = task.get_metadata() + + # Convert to TaskResource schema + # Note: Convert retry/queue to dicts to handle schema differences + task_resource = TaskResource( + id=task_meta.id, + filePath=task_meta.filePath, + exportName=task_meta.exportName, + retry=task_meta.retry.model_dump() if task_meta.retry else None, + queue=task_meta.queue.model_dump() if task_meta.queue else None, + maxDuration=task_meta.maxDuration, + ) + + tasks.append(task_resource.model_dump()) + logger.debug(f"Collected task: {task_id}") + except Exception as e: + logger.error(f"Failed to get metadata for task {task_id}: {e}") + + return tasks + + +async def main(): + """Main indexing workflow""" + logger.info("Python index worker starting") + + # Load manifest + manifest = load_manifest() + logger.info(f"Loaded manifest with {len(manifest.get('tasks', []))} task files") + + # Import all task files + task_files = manifest.get("tasks", []) + success_count = 0 + + for task_file in task_files: + file_path = task_file.get("filePath") or task_file.get("entry") + if file_path and import_task_file(file_path): + success_count += 1 + + logger.info(f"Imported {success_count}/{len(task_files)} task files") + + # Collect task metadata + tasks = collect_task_metadata() + logger.info(f"Found {len(tasks)} tasks") + + # Send INDEX_TASKS_COMPLETE message + ipc = StdioIpcConnection() + message = IndexTasksCompleteMessage(tasks=tasks) + await ipc.send(message) + + logger.info("Indexing complete") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + logger.info("Index worker interrupted") + sys.exit(0) + except Exception as e: + logger.error(f"Index worker failed: {e}", exception=traceback.format_exc()) + sys.exit(1) diff --git a/packages/cli-v3/src/entryPoints/python/managed-run-worker.py b/packages/cli-v3/src/entryPoints/python/managed-run-worker.py new file mode 100755 index 0000000000..fd08c3dc8c --- /dev/null +++ b/packages/cli-v3/src/entryPoints/python/managed-run-worker.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +""" +Python Run Worker + +Executes a single Python task. + +Flow: +1. Receive EXECUTE_TASK_RUN message from stdin +2. Set up execution context (trace context, env vars, etc.) +3. Import task file and get task from registry +4. Execute task with payload +5. Send TASK_RUN_COMPLETED or TASK_RUN_FAILED_TO_RUN message +6. Handle heartbeat and cancellation +""" + +import sys +import json +import os +import asyncio +import importlib.util +import traceback +import signal +from pathlib import Path +from typing import Optional + +# Import SDK (assumes it's installed via pip) +from trigger_sdk.task import TASK_REGISTRY, Task +from trigger_sdk.ipc import StdioIpcConnection +from trigger_sdk.context import TaskContext, set_current_context, clear_current_context +from trigger_sdk.logger import logger +from trigger_sdk.schemas import ExecuteTaskRunMessage + + +# Global state +ipc: Optional[StdioIpcConnection] = None +current_task: Optional[asyncio.Task] = None +cancelled = False + + +def signal_handler(signum, frame): + """Handle termination signals""" + global cancelled + logger.warn(f"Received signal {signum}, cancelling task") + cancelled = True + + if current_task and not current_task.done(): + current_task.cancel() + + +def import_task_file(file_path: str) -> bool: + """Import a Python task file""" + try: + abs_path = Path(file_path).resolve() + + if not abs_path.exists(): + logger.error(f"Task file not found: {file_path}") + return False + + module_name = abs_path.stem.replace(".", "_") + spec = importlib.util.spec_from_file_location(module_name, abs_path) + + if spec is None or spec.loader is None: + logger.error(f"Failed to create module spec for {file_path}") + return False + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + + return True + + except Exception as e: + logger.error(f"Failed to import {file_path}: {e}", exception=traceback.format_exc()) + return False + + +async def heartbeat_loop(run_id: str): + """Send periodic heartbeat messages""" + global ipc, cancelled + + while not cancelled: + await asyncio.sleep(5) # Heartbeat every 5 seconds + if ipc and not cancelled: + try: + await ipc.send_heartbeat(id=run_id) + except Exception as e: + logger.error(f"Failed to send heartbeat: {e}") + + +async def execute_task_run(message: ExecuteTaskRunMessage): + """Execute a task run from the execution message""" + global ipc, current_task, cancelled + + # Parse execution payload using the helper method + execution = message.get_execution() + + task_id = execution.task.id + task_file = execution.task.filePath + run_id = execution.run.id + payload_str = execution.run.payload + + # Parse payload from JSON string to Python object + try: + payload = json.loads(payload_str) if isinstance(payload_str, str) else payload_str + except json.JSONDecodeError as e: + logger.error(f"Failed to parse payload JSON: {e}") + raise ValueError(f"Invalid payload JSON: {e}") + + logger.info(f"Executing task {task_id} from {task_file}") + + # Track start time for usage metrics + import time + start_time = time.time() + + try: + # Import task file if not already imported + if task_id not in TASK_REGISTRY: + if not import_task_file(task_file): + raise RuntimeError(f"Failed to import task file: {task_file}") + + # Get task from registry + if task_id not in TASK_REGISTRY: + raise RuntimeError(f"Task {task_id} not found in registry after import") + + task = TASK_REGISTRY[task_id] + + # Set up execution context + context = TaskContext( + task=execution.task, + run=execution.run, + attempt=execution.attempt, + batch=execution.batch, + environment=execution.environment or {}, + ) + set_current_context(context) + + logger.info(f"Starting task execution (attempt {context.attempt.number})") + + # Start heartbeat with run_id parameter + heartbeat_task = asyncio.create_task(heartbeat_loop(run_id)) + + # Execute task + try: + current_task = asyncio.create_task(task.execute(payload)) + result = await current_task + + # Calculate duration + duration_ms = int((time.time() - start_time) * 1000) + + logger.info("Task completed successfully") + await ipc.send_completed( + id=run_id, + output=result, + usage={"durationMs": duration_ms} + ) + + finally: + # Stop heartbeat + cancelled = True + heartbeat_task.cancel() + try: + await heartbeat_task + except asyncio.CancelledError: + pass + + except asyncio.CancelledError: + duration_ms = int((time.time() - start_time) * 1000) + logger.warn("Task execution cancelled") + await ipc.send_failed( + id=run_id, + error=Exception("Task cancelled"), + usage={"durationMs": duration_ms} + ) + + except Exception as e: + duration_ms = int((time.time() - start_time) * 1000) + logger.error(f"Task execution failed: {e}", exception=traceback.format_exc()) + await ipc.send_failed( + id=run_id, + error=e, + usage={"durationMs": duration_ms} + ) + + finally: + clear_current_context() + current_task = None + + +async def handle_cancel(message): + """Handle cancellation message""" + global current_task + logger.info("Received CANCEL message") + signal_handler(signal.SIGTERM, None) + + +async def main(): + """Main run worker loop""" + global ipc, cancelled + + logger.info("Python run worker starting") + + # Set up signal handlers + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + # Create IPC connection + ipc = StdioIpcConnection() + + # Register message handlers + ipc.on("EXECUTE_TASK_RUN", execute_task_run) + ipc.on("CANCEL", handle_cancel) + + # Start listening for messages + try: + await ipc.start_listening() + except asyncio.CancelledError: + logger.info("Run worker cancelled") + except Exception as e: + logger.error(f"Run worker failed: {e}", exception=traceback.format_exc()) + sys.exit(1) + + logger.info("Run worker stopped") + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + logger.info("Run worker interrupted") + sys.exit(0) + except Exception as e: + logger.error(f"Run worker failed: {e}", exception=traceback.format_exc()) + sys.exit(1) diff --git a/packages/cli-v3/tests/python/test-task.py b/packages/cli-v3/tests/python/test-task.py new file mode 100644 index 0000000000..20a3c5917a --- /dev/null +++ b/packages/cli-v3/tests/python/test-task.py @@ -0,0 +1,30 @@ +"""Test Python task for worker testing""" + +# Import SDK (assumes it's installed via pip) +from trigger_sdk import task, logger + + +@task("test-python-task") +async def test_task(payload): + """Simple test task""" + logger.info(f"Test task received payload: {payload}") + + name = payload.get("name", "World") + + return { + "message": f"Hello {name} from Python!", + "payload": payload, + } + + +@task("test-python-error", retry={"maxAttempts": 3}) +async def error_task(payload): + """Task that raises an error""" + logger.error("This task will fail") + raise RuntimeError("Intentional error for testing") + + +@task("test-python-sync") +def sync_task(payload): + """Synchronous task""" + return {"sync": True, "payload": payload} diff --git a/packages/cli-v3/tests/python/test-workers.sh b/packages/cli-v3/tests/python/test-workers.sh new file mode 100755 index 0000000000..a77f79b289 --- /dev/null +++ b/packages/cli-v3/tests/python/test-workers.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Test script for Python workers + +set -e + +# Set up Python path to find the SDK (for dev mode) +export PYTHONPATH="$(pwd)/packages/python-sdk:$PYTHONPATH" + +echo "Testing Python Index Worker..." + +# Create test manifest +cat > /tmp/test-manifest.json << EOF +{ + "tasks": [ + { + "filePath": "$(pwd)/packages/cli-v3/tests/python/test-task.py" + } + ] +} +EOF + +# Run index worker +export TRIGGER_MANIFEST_PATH=/tmp/test-manifest.json +python3 packages/cli-v3/src/entryPoints/python/managed-index-worker.py > /tmp/index-output.json + +# Verify output +if grep -q "INDEX_TASKS_COMPLETE" /tmp/index-output.json; then + echo "✓ Index worker completed successfully" + cat /tmp/index-output.json | jq '.tasks | length' +else + echo "✗ Index worker failed" + cat /tmp/index-output.json + exit 1 +fi + +echo "" +echo "Testing Python Run Worker..." + +# Create execution message (single-line JSON for line-delimited format) +cat > /tmp/execution.json << 'EOF' +{"type":"EXECUTE_TASK_RUN","version":"v1","execution":{"task":{"id":"test-python-task","filePath":"__TASK_FILE_PATH__","exportName":"test-python-task"},"run":{"id":"run_test123","payload":"{\"name\":\"Test\"}","payloadType":"application/json","tags":[],"isTest":true},"attempt":{"id":"attempt_test123","number":1,"startedAt":"2024-01-01T00:00:00Z","backgroundWorkerId":"worker_test","backgroundWorkerTaskId":"task_test"}}} +EOF + +# Replace placeholder with actual path +sed -i '' "s|__TASK_FILE_PATH__|$(pwd)/packages/cli-v3/tests/python/test-task.py|g" /tmp/execution.json + +# Run execution worker +cat /tmp/execution.json | python3 packages/cli-v3/src/entryPoints/python/managed-run-worker.py > /tmp/run-output.json 2>&1 & +WORKER_PID=$! + +# Wait for completion (max 10 seconds) +for i in {1..10}; do + if ! kill -0 $WORKER_PID 2>/dev/null; then + break + fi + sleep 1 +done + +# Check results +if grep -q "TASK_RUN_COMPLETED" /tmp/run-output.json; then + echo "✓ Run worker completed successfully" + cat /tmp/run-output.json | jq '.completion.output' +else + echo "✗ Run worker failed" + cat /tmp/run-output.json + exit 1 +fi + +echo "" +echo "All worker tests passed!" From 38f2335eba9d7350e07bdac10d2e33ff8997110b Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sun, 9 Nov 2025 00:42:31 +0300 Subject: [PATCH 05/10] feat(python): add stdio IPC bridge for Python workers --- packages/cli-v3/src/python/index.ts | 10 + packages/cli-v3/src/python/pythonProcess.ts | 93 ++++++++ .../cli-v3/src/python/pythonTaskRunner.ts | 104 +++++++++ packages/cli-v3/src/python/stdioIpc.ts | 202 ++++++++++++++++++ .../cli-v3/tests/fixtures/test-manifest.json | 7 + packages/cli-v3/tests/python-ipc.test.ts | 118 ++++++++++ packages/core/src/v3/build/runtime.ts | 15 ++ packages/core/src/v3/schemas/build.ts | 2 +- 8 files changed, 550 insertions(+), 1 deletion(-) create mode 100644 packages/cli-v3/src/python/index.ts create mode 100644 packages/cli-v3/src/python/pythonProcess.ts create mode 100644 packages/cli-v3/src/python/pythonTaskRunner.ts create mode 100644 packages/cli-v3/src/python/stdioIpc.ts create mode 100644 packages/cli-v3/tests/fixtures/test-manifest.json create mode 100644 packages/cli-v3/tests/python-ipc.test.ts diff --git a/packages/cli-v3/src/python/index.ts b/packages/cli-v3/src/python/index.ts new file mode 100644 index 0000000000..e07a883134 --- /dev/null +++ b/packages/cli-v3/src/python/index.ts @@ -0,0 +1,10 @@ +/** + * Python worker infrastructure exports + */ + +export { PythonProcess } from "./pythonProcess.js"; +export { StdioIpcConnection } from "./stdioIpc.js"; +export { PythonTaskRunner } from "./pythonTaskRunner.js"; + +export type { PythonProcessOptions } from "./pythonProcess.js"; +export type { StdioIpcOptions } from "./stdioIpc.js"; diff --git a/packages/cli-v3/src/python/pythonProcess.ts b/packages/cli-v3/src/python/pythonProcess.ts new file mode 100644 index 0000000000..ee35e82861 --- /dev/null +++ b/packages/cli-v3/src/python/pythonProcess.ts @@ -0,0 +1,93 @@ +/** + * Python worker process management. + * + * Spawns Python workers and manages their lifecycle. + */ + +import { spawn, ChildProcess } from "child_process"; +import { BuildRuntime } from "@trigger.dev/core/v3"; +import { StdioIpcConnection } from "./stdioIpc.js"; +import { logger } from "../utilities/logger.js"; +import { execPathForRuntime } from "@trigger.dev/core/v3/build"; + +export interface PythonProcessOptions { + workerScript: string; + cwd?: string; + env?: Record; + runtime?: BuildRuntime; +} + +export class PythonProcess { + private process: ChildProcess | undefined; + private ipc: StdioIpcConnection | undefined; + + constructor(private options: PythonProcessOptions) {} + + async start(): Promise { + const pythonBinary = execPathForRuntime(this.options.runtime ?? "python"); + + logger.debug("Starting Python worker process", { + binary: pythonBinary, + script: this.options.workerScript, + cwd: this.options.cwd, + }); + + this.process = spawn( + pythonBinary, + [ + "-u", // CRITICAL: Unbuffered output for line-delimited JSON IPC + this.options.workerScript, + ], + { + cwd: this.options.cwd ?? process.cwd(), + env: { + ...process.env, + ...this.options.env, + // Ensure unbuffered output + PYTHONUNBUFFERED: "1", + }, + stdio: ["pipe", "pipe", "pipe"], + } + ); + + this.ipc = new StdioIpcConnection({ + process: this.process, + handleStderr: true, + }); + + // Forward logs + this.ipc.on("log", (logData) => { + logger.debug("Python worker log", logData); + }); + + return this.ipc; + } + + async kill(signal: NodeJS.Signals = "SIGTERM"): Promise { + if (!this.process) return; + + return new Promise((resolve) => { + const timeout = setTimeout(() => { + logger.warn("Python worker did not exit gracefully, forcing kill"); + this.process?.kill("SIGKILL"); + resolve(); + }, 5000); + + this.process.once("exit", () => { + clearTimeout(timeout); + resolve(); + }); + + this.process.kill(signal); + }); + } + + async cleanup(): Promise { + this.ipc?.close(); + await this.kill(); + } + + get pid(): number | undefined { + return this.process?.pid; + } +} diff --git a/packages/cli-v3/src/python/pythonTaskRunner.ts b/packages/cli-v3/src/python/pythonTaskRunner.ts new file mode 100644 index 0000000000..49b9d127d6 --- /dev/null +++ b/packages/cli-v3/src/python/pythonTaskRunner.ts @@ -0,0 +1,104 @@ +/** + * High-level Python task execution. + * + * Manages Python worker lifecycle for task execution. + */ + +import path from "path"; +import { fileURLToPath } from "url"; +import { PythonProcess } from "./pythonProcess.js"; +import { TaskRunExecution, TaskRunExecutionResult } from "@trigger.dev/core/v3"; +import { logger } from "../utilities/logger.js"; + +// Get __dirname equivalent in ESM +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +export class PythonTaskRunner { + async executeTask(execution: TaskRunExecution): Promise { + const workerScript = path.join(__dirname, "../entryPoints/python/managed-run-worker.py"); + + const pythonProcess = new PythonProcess({ + workerScript, + env: { + TRIGGER_MANIFEST_PATH: execution.worker.manifestPath, + // Add SDK path for dev mode (assumes SDK is in packages/python-sdk) + PYTHONPATH: path.join(__dirname, "../../../python-sdk"), + // Add trace context, env vars, etc. + }, + }); + + try { + const ipc = await pythonProcess.start(); + + // Wait for completion or failure + const result = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error("Task execution timeout")); + }, execution.task.maxDuration ?? 300000); + + ipc.on("TASK_RUN_COMPLETED", (message: any) => { + clearTimeout(timeout); + resolve({ + ok: true, + output: message.completion.output, + outputType: message.completion.outputType, + usage: message.completion.usage, + }); + }); + + ipc.on("TASK_RUN_FAILED_TO_RUN", (message: any) => { + clearTimeout(timeout); + resolve({ + ok: false, + error: message.completion.error, + usage: message.completion.usage, + }); + }); + + ipc.on("TASK_HEARTBEAT", () => { + logger.debug("Received heartbeat from Python task"); + }); + + ipc.on("exit", (code: number | null) => { + clearTimeout(timeout); + if (code !== 0) { + reject(new Error(`Python worker exited with code ${code}`)); + } + }); + + // Send execution message + ipc.send({ + type: "EXECUTE_TASK_RUN", + version: "v1", + execution: { + task: { + id: execution.task.id, + filePath: execution.task.filePath, + exportName: execution.task.exportName, + }, + run: { + id: execution.run.id, + payload: JSON.stringify(execution.run.payload), // CRITICAL: Must be JSON string + payloadType: execution.run.payloadType, + context: execution.run.context, + tags: execution.run.tags, + isTest: execution.run.isTest, + }, + attempt: { + id: execution.attempt.id, + number: execution.attempt.number, + startedAt: execution.attempt.startedAt, + backgroundWorkerId: execution.attempt.backgroundWorkerId, + backgroundWorkerTaskId: execution.attempt.backgroundWorkerTaskId, + }, + }, + }); + }); + + return result; + } finally { + await pythonProcess.cleanup(); + } + } +} diff --git a/packages/cli-v3/src/python/stdioIpc.ts b/packages/cli-v3/src/python/stdioIpc.ts new file mode 100644 index 0000000000..056355ae25 --- /dev/null +++ b/packages/cli-v3/src/python/stdioIpc.ts @@ -0,0 +1,202 @@ +/** + * Stdio-based IPC connection for Python workers. + * + * Communicates with Python processes via line-delimited JSON over stdio. + * Compatible with Python's StdioIpcConnection implementation. + */ + +import { ChildProcess } from "child_process"; +import readline from "readline"; +import { z } from "zod"; +import { EventEmitter } from "events"; +import { logger } from "../utilities/logger.js"; + +// Message schemas matching Python Pydantic schemas +const TaskRunCompletedSchema = z.object({ + type: z.literal("TASK_RUN_COMPLETED"), + version: z.literal("v1"), + completion: z.object({ + ok: z.literal(true), + id: z.string(), + output: z.string(), + outputType: z.string(), + usage: z + .object({ + durationMs: z.number().optional(), + }) + .optional(), + }), +}); + +const TaskRunFailedSchema = z.object({ + type: z.literal("TASK_RUN_FAILED_TO_RUN"), + version: z.literal("v1"), + completion: z.object({ + ok: z.literal(false), + id: z.string(), + error: z.object({ + type: z.string(), + message: z.string(), + stackTrace: z.string().optional(), + }), + usage: z + .object({ + durationMs: z.number().optional(), + }) + .optional(), + }), +}); + +const TaskHeartbeatSchema = z.object({ + type: z.literal("TASK_HEARTBEAT"), + version: z.literal("v1"), + id: z.string(), +}); + +const IndexCompleteSchema = z.object({ + type: z.literal("INDEX_TASKS_COMPLETE"), + version: z.literal("v1"), + tasks: z.array(z.record(z.any())), +}); + +const WorkerMessageSchema = z.discriminatedUnion("type", [ + TaskRunCompletedSchema, + TaskRunFailedSchema, + TaskHeartbeatSchema, + IndexCompleteSchema, +]); + +type WorkerMessage = z.infer; + +export interface StdioIpcOptions { + process: ChildProcess; + handleStderr?: boolean; +} + +export class StdioIpcConnection extends EventEmitter { + private process: ChildProcess; + private stdoutReader: readline.Interface | undefined; + private stderrReader: readline.Interface | undefined; + private closed = false; + + constructor(options: StdioIpcOptions) { + super(); + this.process = options.process; + + // Set up stdout reader for IPC messages + if (this.process.stdout) { + this.stdoutReader = readline.createInterface({ + input: this.process.stdout, + crlfDelay: Infinity, + }); + + this.stdoutReader.on("line", (line) => this.handleStdoutLine(line)); + } + + // Set up stderr reader for logs (optional) + if (options.handleStderr && this.process.stderr) { + this.stderrReader = readline.createInterface({ + input: this.process.stderr, + crlfDelay: Infinity, + }); + + this.stderrReader.on("line", (line) => this.handleStderrLine(line)); + } + + // Handle process exit + this.process.on("exit", (code, signal) => { + this.handleProcessExit(code, signal); + }); + + this.process.on("error", (error) => { + this.emit("error", error); + }); + } + + private handleStdoutLine(line: string) { + if (!line.trim()) return; + + try { + const data = JSON.parse(line); + const message = WorkerMessageSchema.parse(data); + + logger.debug("Received message from Python worker", { + type: message.type, + message, + }); + + this.emit("message", message); + this.emit(message.type, message); + } catch (error) { + logger.error("Failed to parse Python worker message", { + line, + error: error instanceof Error ? error.message : String(error), + }); + this.emit("parseError", error); + } + } + + private handleStderrLine(line: string) { + if (!line.trim()) return; + + try { + // Try to parse as structured log + const logData = JSON.parse(line); + this.emit("log", logData); + } catch { + // Plain text log + this.emit("log", { message: line, level: "INFO" }); + } + } + + private handleProcessExit(code: number | null, signal: NodeJS.Signals | null) { + if (this.closed) return; + + logger.debug("Python worker process exited", { code, signal }); + + this.close(); + this.emit("exit", code, signal); + } + + send(message: Record) { + if (this.closed) { + throw new Error("Cannot send message: IPC connection closed"); + } + + if (!this.process.stdin) { + throw new Error("Process stdin not available"); + } + + try { + const json = JSON.stringify(message); + this.process.stdin.write(json + "\n"); + + logger.debug("Sent message to Python worker", { + type: message.type, + message, + }); + } catch (error) { + logger.error("Failed to send message to Python worker", { error }); + throw error; + } + } + + close() { + if (this.closed) return; + + this.closed = true; + + this.stdoutReader?.close(); + this.stderrReader?.close(); + + if (this.process.stdin) { + this.process.stdin.end(); + } + + this.removeAllListeners(); + } + + get isClosed() { + return this.closed; + } +} diff --git a/packages/cli-v3/tests/fixtures/test-manifest.json b/packages/cli-v3/tests/fixtures/test-manifest.json new file mode 100644 index 0000000000..ef11787848 --- /dev/null +++ b/packages/cli-v3/tests/fixtures/test-manifest.json @@ -0,0 +1,7 @@ +{ + "tasks": [ + { + "filePath": "packages/cli-v3/tests/python/test-task.py" + } + ] +} diff --git a/packages/cli-v3/tests/python-ipc.test.ts b/packages/cli-v3/tests/python-ipc.test.ts new file mode 100644 index 0000000000..d307e29763 --- /dev/null +++ b/packages/cli-v3/tests/python-ipc.test.ts @@ -0,0 +1,118 @@ +import { describe, it, expect } from "vitest"; +import { PythonProcess } from "../src/python/pythonProcess.js"; +import path from "path"; +import { fileURLToPath } from "url"; + +// Get __dirname equivalent in ESM +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +describe("Python IPC", () => { + it("can spawn Python worker and communicate", async () => { + const indexWorker = path.join(__dirname, "../src/entryPoints/python/managed-index-worker.py"); + + const manifestPath = path.join(__dirname, "fixtures/test-manifest.json"); + + const pythonProcess = new PythonProcess({ + workerScript: indexWorker, + env: { + TRIGGER_MANIFEST_PATH: manifestPath, + PYTHONPATH: path.join(__dirname, "../../python-sdk"), + }, + }); + + const ipc = await pythonProcess.start(); + + const result = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error("Timeout")), 10000); + + ipc.on("INDEX_TASKS_COMPLETE", (message: any) => { + clearTimeout(timeout); + resolve(message); + }); + + ipc.on("error", (error: Error) => { + clearTimeout(timeout); + reject(error); + }); + + ipc.on("exit", (code: number | null) => { + clearTimeout(timeout); + if (code !== 0) { + reject(new Error(`Process exited with code ${code}`)); + } + }); + }); + + expect(result).toHaveProperty("tasks"); + expect((result as any).tasks.length).toBeGreaterThan(0); + + await pythonProcess.cleanup(); + }); + + it("can execute Python task end-to-end", async () => { + const runWorker = path.join(__dirname, "../src/entryPoints/python/managed-run-worker.py"); + + const pythonProcess = new PythonProcess({ + workerScript: runWorker, + env: { + PYTHONPATH: path.join(__dirname, "../../python-sdk"), + }, + }); + + const ipc = await pythonProcess.start(); + + const result = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error("Timeout")), 10000); + + ipc.on("TASK_RUN_COMPLETED", (message: any) => { + clearTimeout(timeout); + resolve(message); + }); + + ipc.on("TASK_RUN_FAILED_TO_RUN", (message: any) => { + clearTimeout(timeout); + reject(new Error(`Task failed: ${message.completion.error.message}`)); + }); + + ipc.on("error", (error: Error) => { + clearTimeout(timeout); + reject(error); + }); + + // Send execution message + ipc.send({ + type: "EXECUTE_TASK_RUN", + version: "v1", + execution: { + task: { + id: "test-python-task", + filePath: path.join(__dirname, "python/test-task.py"), + exportName: "test-python-task", + }, + run: { + id: "run_test123", + payload: JSON.stringify({ name: "Test" }), + payloadType: "application/json", + context: {}, + tags: [], + isTest: true, + }, + attempt: { + id: "attempt_test123", + number: 1, + startedAt: new Date().toISOString(), + backgroundWorkerId: "worker_test", + backgroundWorkerTaskId: "task_test", + }, + }, + }); + }); + + expect(result.type).toBe("TASK_RUN_COMPLETED"); + expect(result.completion.ok).toBe(true); + expect(result.completion.output).toContain("Hello"); + + await pythonProcess.cleanup(); + }); +}); diff --git a/packages/core/src/v3/build/runtime.ts b/packages/core/src/v3/build/runtime.ts index 1618a50ffd..67771730e3 100644 --- a/packages/core/src/v3/build/runtime.ts +++ b/packages/core/src/v3/build/runtime.ts @@ -13,6 +13,8 @@ export function binaryForRuntime(runtime: BuildRuntime): string { return "node"; case "bun": return "bun"; + case "python": + return "python3"; default: throw new Error(`Unsupported runtime ${runtime}`); } @@ -33,6 +35,15 @@ export function execPathForRuntime(runtime: BuildRuntime): string { } return join(homedir(), ".bun", "bin", "bun"); + case "python": { + // Check for custom Python path + if (typeof process.env.PYTHON_BIN_PATH === "string") { + return process.env.PYTHON_BIN_PATH; + } + + // Default to python3 in PATH + return "python3"; + } default: throw new Error(`Unsupported runtime ${runtime}`); } @@ -74,6 +85,10 @@ export function execOptionsForRuntime( case "bun": { return ""; } + case "python": { + // CRITICAL: -u flag ensures unbuffered stdout for line-delimited JSON IPC + return "-u"; + } } } diff --git a/packages/core/src/v3/schemas/build.ts b/packages/core/src/v3/schemas/build.ts index ee33bb7efb..ec16f3467e 100644 --- a/packages/core/src/v3/schemas/build.ts +++ b/packages/core/src/v3/schemas/build.ts @@ -13,7 +13,7 @@ export const BuildTarget = z.enum(["dev", "deploy", "unmanaged"]); export type BuildTarget = z.infer; -export const BuildRuntime = z.enum(["node", "node-22", "bun"]); +export const BuildRuntime = z.enum(["node", "node-22", "bun", "python"]); export type BuildRuntime = z.infer; From 5d1a617492e3089e391c2f86bab260e6c4a60461 Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sun, 9 Nov 2025 01:20:42 +0300 Subject: [PATCH 06/10] feat(cli): integrate Python runtime into task execution --- .../src/entryPoints/dev-run-controller.ts | 61 ++++++++++++++++++- .../src/entryPoints/managed/execution.ts | 26 ++++++++ .../cli-v3/src/python/pythonTaskRunner.ts | 11 +++- packages/cli-v3/tests/python-runtime.test.ts | 17 ++++++ packages/core/src/v3/build/runtime.ts | 9 +-- 5 files changed, 113 insertions(+), 11 deletions(-) create mode 100644 packages/cli-v3/tests/python-runtime.test.ts diff --git a/packages/cli-v3/src/entryPoints/dev-run-controller.ts b/packages/cli-v3/src/entryPoints/dev-run-controller.ts index 12876b3240..367066bbad 100644 --- a/packages/cli-v3/src/entryPoints/dev-run-controller.ts +++ b/packages/cli-v3/src/entryPoints/dev-run-controller.ts @@ -21,6 +21,7 @@ import { join } from "node:path"; import { BackgroundWorker } from "../dev/backgroundWorker.js"; import { eventBus } from "../utilities/eventBus.js"; import { TaskRunProcessPool } from "../dev/taskRunProcessPool.js"; +import { PythonTaskRunner } from "../python/index.js"; type DevRunControllerOptions = { runFriendlyId: string; @@ -597,7 +598,65 @@ export class DevRunController { this.isCompletingRun = false; - // Get process from pool instead of creating new one + // Check if this is a Python task - use PythonTaskRunner instead of TaskRunProcess + if (this.opts.worker.manifest.runtime === "python") { + logger.debug("Executing Python task", { taskId: execution.task.id }); + + const pythonRunner = new PythonTaskRunner(); + const completion = await pythonRunner.executeTask({ + ...execution, + worker: { + runtime: "python", + manifestPath: join(this.opts.worker.build.outputPath, "index.json"), + }, + }); + + logger.debug("Completed Python run", completion); + + this.isCompletingRun = true; + + if (!this.runFriendlyId || !this.snapshotFriendlyId) { + logger.debug("executeRun: Missing run ID or snapshot ID after Python execution", { + runId: this.runFriendlyId, + snapshotId: this.snapshotFriendlyId, + }); + + this.runFinished(); + return; + } + + const completionResult = await this.httpClient.dev.completeRunAttempt( + this.runFriendlyId, + this.snapshotFriendlyId, + { + completion, + } + ); + + if (!completionResult.success) { + logger.debug("Failed to submit Python completion", { + error: completionResult.error, + }); + + this.runFinished(); + return; + } + + logger.debug("Python attempt completion submitted", completionResult.data.result); + + try { + await this.handleCompletionResult(completion, completionResult.data.result, execution); + } catch (error) { + logger.debug("Failed to handle Python completion result", { error }); + + this.runFinished(); + return; + } + + return; + } + + // Get process from pool instead of creating new one (Node.js/Bun) const { taskRunProcess, isReused } = await this.opts.taskRunProcessPool.getProcess( this.opts.worker.manifest, { diff --git a/packages/cli-v3/src/entryPoints/managed/execution.ts b/packages/cli-v3/src/entryPoints/managed/execution.ts index 2dd3e6838e..b66f172f4e 100644 --- a/packages/cli-v3/src/entryPoints/managed/execution.ts +++ b/packages/cli-v3/src/entryPoints/managed/execution.ts @@ -23,6 +23,7 @@ import { SnapshotManager, SnapshotState } from "./snapshot.js"; import type { SupervisorSocket } from "./controller.js"; import { RunNotifier } from "./notifier.js"; import { TaskRunProcessProvider } from "./taskRunProcessProvider.js"; +import { PythonTaskRunner } from "../../python/index.js"; class ExecutionAbortError extends Error { constructor(message: string) { @@ -609,6 +610,31 @@ export class RunExecution { const taskRunEnv = this.currentTaskRunEnv ?? envVars; + // Check if this is a Python task - use PythonTaskRunner instead of TaskRunProcess + if (this.opts.workerManifest.runtime === "python") { + this.sendDebugLog("executing Python task", { taskId: execution.task.id }); + + const pythonRunner = new PythonTaskRunner(); + const completion = await pythonRunner.executeTask({ + ...execution, + worker: { + runtime: "python", + manifestPath: this.env.TRIGGER_WORKER_MANIFEST_PATH, + }, + }); + + this.sendDebugLog("completed Python run attempt", { attemptSuccess: completion.ok }); + + const [completionError] = await tryCatch(this.complete({ completion })); + + if (completionError) { + this.sendDebugLog("failed to complete Python run", { error: completionError.message }); + } + + return; + } + + // Node.js/Bun execution path if (!this.taskRunProcess || this.taskRunProcess.isBeingKilled) { this.sendDebugLog("getting new task run process", { runId: execution.run.id }); this.taskRunProcess = await this.taskRunProcessProvider.getProcess({ diff --git a/packages/cli-v3/src/python/pythonTaskRunner.ts b/packages/cli-v3/src/python/pythonTaskRunner.ts index 49b9d127d6..2e6e1acdd3 100644 --- a/packages/cli-v3/src/python/pythonTaskRunner.ts +++ b/packages/cli-v3/src/python/pythonTaskRunner.ts @@ -18,12 +18,19 @@ export class PythonTaskRunner { async executeTask(execution: TaskRunExecution): Promise { const workerScript = path.join(__dirname, "../entryPoints/python/managed-run-worker.py"); + // Determine PYTHONPATH - use env var if set, otherwise use relative path for dev + const pythonPath = process.env.TRIGGER_PYTHON_SDK_PATH + ? process.env.TRIGGER_PYTHON_SDK_PATH + : path.join(__dirname, "../../../python-sdk"); + const pythonProcess = new PythonProcess({ workerScript, env: { TRIGGER_MANIFEST_PATH: execution.worker.manifestPath, - // Add SDK path for dev mode (assumes SDK is in packages/python-sdk) - PYTHONPATH: path.join(__dirname, "../../../python-sdk"), + // Add SDK path for dev mode (in production, SDK is installed via pip) + ...(process.env.TRIGGER_PYTHON_SDK_PATH || process.env.NODE_ENV !== "production" + ? { PYTHONPATH: pythonPath } + : {}), // Add trace context, env vars, etc. }, }); diff --git a/packages/cli-v3/tests/python-runtime.test.ts b/packages/cli-v3/tests/python-runtime.test.ts new file mode 100644 index 0000000000..27adbc3006 --- /dev/null +++ b/packages/cli-v3/tests/python-runtime.test.ts @@ -0,0 +1,17 @@ +import { describe, it, expect } from "vitest"; +import { + execPathForRuntime, + execOptionsForRuntime, +} from "@trigger.dev/core/v3/build"; + +describe("Python Runtime", () => { + it("returns python3 binary path", () => { + const pythonPath = execPathForRuntime("python"); + expect(pythonPath).toBe("python3"); + }); + + it("provides Python exec options with unbuffered flag", () => { + const options = execOptionsForRuntime("python", {}); + expect(options).toBe("-u"); + }); +}); diff --git a/packages/core/src/v3/build/runtime.ts b/packages/core/src/v3/build/runtime.ts index 67771730e3..33b2364ca4 100644 --- a/packages/core/src/v3/build/runtime.ts +++ b/packages/core/src/v3/build/runtime.ts @@ -35,15 +35,8 @@ export function execPathForRuntime(runtime: BuildRuntime): string { } return join(homedir(), ".bun", "bin", "bun"); - case "python": { - // Check for custom Python path - if (typeof process.env.PYTHON_BIN_PATH === "string") { - return process.env.PYTHON_BIN_PATH; - } - - // Default to python3 in PATH + case "python": return "python3"; - } default: throw new Error(`Unsupported runtime ${runtime}`); } From d1a01dc501439f6503dab62cdf64c00a165b8f8d Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sun, 9 Nov 2025 01:55:44 +0300 Subject: [PATCH 07/10] feat(python): add Python build system support --- packages/cli-v3/src/build/bundle.ts | 96 +++++++- packages/cli-v3/src/build/entryPoints.ts | 36 ++- packages/cli-v3/src/build/pythonBundler.ts | 196 ++++++++++++++++ .../cli-v3/src/build/pythonDependencies.ts | 103 +++++++++ .../cli-v3/tests/python-dependencies.test.ts | 210 ++++++++++++++++++ packages/core/src/v3/schemas/build.ts | 1 + packages/core/src/v3/schemas/schemas.ts | 1 + 7 files changed, 635 insertions(+), 8 deletions(-) create mode 100644 packages/cli-v3/src/build/pythonBundler.ts create mode 100644 packages/cli-v3/src/build/pythonDependencies.ts create mode 100644 packages/cli-v3/tests/python-dependencies.test.ts diff --git a/packages/cli-v3/src/build/bundle.ts b/packages/cli-v3/src/build/bundle.ts index 597b46854a..1f8a8ee34a 100644 --- a/packages/cli-v3/src/build/bundle.ts +++ b/packages/cli-v3/src/build/bundle.ts @@ -3,7 +3,8 @@ import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build"; import { BuildManifest, BuildTarget, TaskFile } from "@trigger.dev/core/v3/schemas"; import * as esbuild from "esbuild"; import { createHash } from "node:crypto"; -import { join, relative, resolve } from "node:path"; +import path, { join, relative, resolve } from "node:path"; +import fs from "fs-extra"; import { createFile } from "../utilities/fileSystem.js"; import { logger } from "../utilities/logger.js"; import { resolveFileSources } from "../utilities/sourceFiles.js"; @@ -26,6 +27,7 @@ import { import { buildPlugins } from "./plugins.js"; import { cliLink, prettyError } from "../utilities/cliOutput.js"; import { SkipLoggingError } from "../cli/common.js"; +import { bundlePython, createBuildManifestFromPythonBundle } from "./pythonBundler.js"; export interface BundleOptions { target: BuildTarget; @@ -65,6 +67,11 @@ export class BundleError extends Error { export async function bundleWorker(options: BundleOptions): Promise { const { resolvedConfig } = options; + // Handle Python runtime + if (resolvedConfig.runtime === "python") { + return bundlePythonWorker(options); + } + let currentContext: esbuild.BuildContext | undefined; const entryPointManager = await createEntryPointManager( @@ -405,3 +412,90 @@ export async function createBuildManifestFromBundle({ return copyManifestToDir(buildManifest, destination, workerDir); } + +/** + * Bundle Python worker - entry point for Python runtime. + * This is the Python equivalent of bundleWorker for Node.js. + */ +async function bundlePythonWorker(options: BundleOptions): Promise { + const { resolvedConfig, destination, cwd, target } = options; + + const entryPointManager = await createEntryPointManager( + resolvedConfig.dirs, + resolvedConfig, + options.target, + typeof options.watch === "boolean" ? options.watch : false, + async (newEntryPoints) => { + // TODO: Implement proper watch mode for Python (file copying + manifest regeneration) + logger.debug("Python entry points changed, rebuilding"); + } + ); + + if (entryPointManager.entryPoints.length === 0) { + const errorMessageBody = ` + Dirs config: + ${resolvedConfig.dirs.join("\n- ")} + + Search patterns: + ${entryPointManager.patterns.join("\n- ")} + + Possible solutions: + 1. Check if the directory paths in your config are correct + 2. Verify that your files match the search patterns + 3. Update your trigger.config.ts runtime to "python" + `.replace(/^ {6}/gm, ""); + + prettyError( + "No Python task files found", + errorMessageBody, + cliLink("View the config docs", "https://trigger.dev/docs/config/config-file") + ); + + throw new SkipLoggingError(); + } + + // Bundle Python files + logger.debug("Starting Python bundle", { + entryPoints: entryPointManager.entryPoints.length, + }); + + const bundleResult = await bundlePython({ + entryPoints: entryPointManager.entryPoints, + outputDir: destination, + projectDir: cwd, + requirementsFile: process.env.TRIGGER_REQUIREMENTS_FILE, + config: resolvedConfig, + target, + }); + + // Create complete BuildManifest + const buildManifest = await createBuildManifestFromPythonBundle(bundleResult, { + outputDir: destination, + projectDir: cwd, + config: resolvedConfig, + target, + }); + + // Write manifest to output + const manifestPath = join(destination, "build-manifest.json"); + await fs.writeJson(manifestPath, buildManifest, { spaces: 2 }); + + // Convert to BundleResult + const pythonBundleResult: BundleResult = { + contentHash: buildManifest.contentHash, + files: buildManifest.files, + configPath: buildManifest.configPath, + metafile: {} as esbuild.Metafile, // Empty for Python + loaderEntryPoint: undefined, + runWorkerEntryPoint: buildManifest.runWorkerEntryPoint, + runControllerEntryPoint: undefined, + indexWorkerEntryPoint: buildManifest.indexWorkerEntryPoint, + indexControllerEntryPoint: undefined, + initEntryPoint: undefined, + stop: async () => { + await entryPointManager.stop(); + }, + }; + + return pythonBundleResult; +} diff --git a/packages/cli-v3/src/build/entryPoints.ts b/packages/cli-v3/src/build/entryPoints.ts index c738a16ca5..905c23d3f8 100644 --- a/packages/cli-v3/src/build/entryPoints.ts +++ b/packages/cli-v3/src/build/entryPoints.ts @@ -33,6 +33,18 @@ const DEFAULT_IGNORE_PATTERNS = [ "**/*.spec.cjs", ]; +const PYTHON_IGNORE_PATTERNS = [ + "**/__pycache__/**", + "**/venv/**", + "**/.venv/**", + "**/.pytest_cache/**", + "**/.mypy_cache/**", + "**/*.test.py", + "**/*.spec.py", + "**/test_*.py", + "**/tests/**", +]; + export async function createEntryPointManager( dirs: string[], config: ResolvedConfig, @@ -40,17 +52,27 @@ export async function createEntryPointManager( watch: boolean, onEntryPointsChange?: (entryPoints: string[]) => Promise ): Promise { + // Determine file extension patterns based on runtime + const fileExtensions = + config.runtime === "python" + ? ["*.py"] + : ["*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}"]; + // Patterns to match files - const patterns = dirs.flatMap((dir) => [ - `${ - isDynamicPattern(dir) - ? `${dir}/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}` - : `${escapePath(dir)}/**/*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}` - }`, - ]); + const patterns = dirs.flatMap((dir) => + fileExtensions.map((ext) => + isDynamicPattern(dir) ? `${dir}/${ext}` : `${escapePath(dir)}/**/${ext}` + ) + ); // Patterns to ignore let ignorePatterns = config.ignorePatterns ?? DEFAULT_IGNORE_PATTERNS; + + // Add Python-specific ignore patterns if runtime is python + if (config.runtime === "python") { + ignorePatterns = ignorePatterns.concat(PYTHON_IGNORE_PATTERNS); + } + ignorePatterns = ignorePatterns.concat([ "**/node_modules/**", "**/.git/**", diff --git a/packages/cli-v3/src/build/pythonBundler.ts b/packages/cli-v3/src/build/pythonBundler.ts new file mode 100644 index 0000000000..e8c8e978ce --- /dev/null +++ b/packages/cli-v3/src/build/pythonBundler.ts @@ -0,0 +1,196 @@ +/** + * Python bundler - copies Python files and generates build manifest. + * + * Unlike Node.js which uses esbuild, Python files are copied as-is. + */ + +import path from "path"; +import fs from "fs-extra"; +import { createHash } from "node:crypto"; +import type { BuildManifest, BuildTarget } from "@trigger.dev/core/v3"; +import type { ResolvedConfig } from "@trigger.dev/core/v3/build"; +import { logger } from "../utilities/logger.js"; +import { parseRequirementsTxt } from "./pythonDependencies.js"; +import { VERSION } from "../version.js"; +import { CORE_VERSION } from "@trigger.dev/core/v3"; + +export interface PythonBundleOptions { + entryPoints: string[]; // Absolute paths to Python files + outputDir: string; // Build output directory + projectDir: string; // Project root + requirementsFile?: string; // Optional path to requirements.txt + config: ResolvedConfig; // Resolved config + target: BuildTarget; // dev, deploy, or unmanaged +} + +export interface PythonBundleResult { + entries: Array<{ + entry: string; // Absolute path to input file + out: string; // Relative path to output file + relativePath: string; // Relative path from project dir + content: string; // File content + contentHash: string; // Content hash + }>; + requirementsContent?: string; +} + +/** + * Bundle Python tasks by copying files and generating manifest. + * Returns data needed to create BuildManifest. + */ +export async function bundlePython(options: PythonBundleOptions): Promise { + const { entryPoints, outputDir, projectDir, requirementsFile, config, target } = options; + + logger.info("Bundling Python tasks", { + entryPoints: entryPoints.length, + outputDir, + projectDir, + }); + + // Create output directory + await fs.ensureDir(outputDir); + + // Copy Python files to output + const entries: PythonBundleResult["entries"] = []; + + for (const entryPoint of entryPoints) { + if (entryPoint.endsWith(".py")) { + const relativePath = path.relative(projectDir, entryPoint); + const outputPath = path.join(outputDir, relativePath); + + // Ensure output directory exists + await fs.ensureDir(path.dirname(outputPath)); + + // Read file content + const content = await fs.readFile(entryPoint, "utf-8"); + + // Copy file + await fs.copyFile(entryPoint, outputPath); + + // Calculate content hash + const contentHash = createHash("md5").update(content).digest("hex"); + + logger.debug("Copied Python task file", { + from: entryPoint, + to: outputPath, + relativePath, + size: content.length, + }); + + entries.push({ + entry: entryPoint, + out: outputPath, + relativePath, + content, + contentHash, + }); + } + } + + // Read requirements.txt if provided or look for default + let requirementsContent: string | undefined; + const reqPath = requirementsFile || path.join(projectDir, "requirements.txt"); + + if (await fs.pathExists(reqPath)) { + requirementsContent = await fs.readFile(reqPath, "utf-8"); + + // Copy requirements.txt to output + await fs.copyFile(reqPath, path.join(outputDir, "requirements.txt")); + + logger.info("Copied requirements.txt", { + path: reqPath, + dependencies: parseRequirementsTxt(requirementsContent).length, + }); + } else { + logger.warn("No requirements.txt found, Python tasks may have missing dependencies"); + } + + const result: PythonBundleResult = { + entries, + requirementsContent, + }; + + logger.info("Python bundle complete", { + files: entries.length, + requirements: requirementsContent ? parseRequirementsTxt(requirementsContent).length : 0, + outputDir, + }); + + return result; +} + +/** + * Generate BuildManifest from bundle result. + */ +export async function createBuildManifestFromPythonBundle( + bundle: PythonBundleResult, + options: Pick +): Promise { + const { outputDir, config, target } = options; + + // Create config manifest (same as Node.js projects) + const configManifest = { + project: config.project, + dirs: config.dirs, + }; + + // TODO: Get environment from CLI options (like Node.js build does) + const environment = config.deploy?.env?.ENVIRONMENT ?? "development"; + // TODO: Get branch from git or CLI options + const branch = undefined; + // TODO: Get sdkVersion from CLI or package.json + const sdkVersion = CORE_VERSION; + + // Calculate overall content hash from all file hashes + const hasher = createHash("md5"); + for (const entry of bundle.entries) { + hasher.update(entry.contentHash); + } + const contentHash = hasher.digest("hex"); + + // Build sources map (file path -> content + hash) + const sources: Record = {}; + for (const entry of bundle.entries) { + sources[entry.relativePath] = { + contents: entry.content, + contentHash: entry.contentHash, + }; + } + + // Build files array + const files = bundle.entries.map((entry) => ({ + entry: entry.entry, + out: entry.out, + filePath: entry.relativePath, + })); + + const buildManifest: BuildManifest = { + target, + packageVersion: sdkVersion ?? CORE_VERSION, + cliPackageVersion: VERSION, + contentHash, + runtime: "python", + environment, + branch, + config: configManifest, + files, + sources, + outputPath: outputDir, + // Python entry points - these are relative paths to Python scripts + // The runtime will resolve them from the CLI package + runWorkerEntryPoint: "entryPoints/python/managed-run-worker.py", + indexWorkerEntryPoint: "entryPoints/python/managed-index-worker.py", + configPath: config.configFile || "trigger.config.ts", + build: {}, + deploy: { + env: {}, + }, + customConditions: config.build?.conditions ?? [], + otelImportHook: { + include: config.instrumentedPackageNames ?? [], + }, + requirementsContent: bundle.requirementsContent, + }; + + return buildManifest; +} diff --git a/packages/cli-v3/src/build/pythonDependencies.ts b/packages/cli-v3/src/build/pythonDependencies.ts new file mode 100644 index 0000000000..af081b300d --- /dev/null +++ b/packages/cli-v3/src/build/pythonDependencies.ts @@ -0,0 +1,103 @@ +/** + * Parse and manage Python dependencies. + */ + +export interface PythonDependency { + name: string; + version?: string; + extras?: string[]; +} + +/** + * Parse requirements.txt content into structured dependencies. + * Supports syntax: package[extra1,extra2]==version + * Package names must follow PEP 508: start with letter or underscore, followed by letters, digits, hyphens, or underscores + */ +export function parseRequirementsTxt(content: string): PythonDependency[] { + const lines = content.split("\n"); + const dependencies: PythonDependency[] = []; + + for (const line of lines) { + const trimmed = line.trim(); + + // Skip empty lines and comments + if (!trimmed || trimmed.startsWith("#")) continue; + + // Parse dependency in format: package[extra1,extra2]==version + // Full regex for package name: [A-Za-z_][A-Za-z0-9_-]* + const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_-]*)(?:\[([^\]]+)\])?(?:([<>=!~]+)(.+))?$/); + + if (match) { + const [, name, extras, operator, version] = match; + + dependencies.push({ + name: name.trim(), + extras: extras?.split(",").map((e) => e.trim()), + version: version ? `${operator}${version.trim()}` : undefined, + }); + } else { + // Try simple package name (reusing the same regex pattern) + const simpleMatch = trimmed.match(/^([A-Za-z_][A-Za-z0-9_-]*)/); + if (simpleMatch) { + dependencies.push({ + name: simpleMatch[1], + }); + } + } + } + + return dependencies; +} + +/** + * Generate requirements.txt content from dependency objects. + */ +export function generateRequirementsTxt(dependencies: PythonDependency[]): string { + return dependencies + .map((dep) => { + let line = dep.name; + if (dep.extras?.length) { + line += `[${dep.extras.join(",")}]`; + } + if (dep.version) { + line += dep.version; + } + return line; + }) + .join("\n"); +} + +/** + * Validate requirements.txt syntax. + * Returns parsing errors if any. + */ +export function validateRequirementsTxt(content: string): { valid: boolean; errors: string[] } { + const lines = content.split("\n"); + const errors: string[] = []; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i].trim(); + + // Skip empty lines and comments + if (!line || line.startsWith("#")) continue; + + // Try to parse the line + try { + const match = line.match(/^(\w[\w\-]*)(?:\[([^\]]+)\])?(?:([<>=!~]+)(.+))?$/); + if (!match) { + // Try simple package name + const simpleMatch = line.match(/^(\w[\w\-]*)/); + if (!simpleMatch) { + errors.push(`Line ${i + 1}: Invalid format "${line}"`); + } + } + } catch (error) { + errors.push(`Line ${i + 1}: ${error instanceof Error ? error.message : String(error)}`); + } + } + + return { + valid: errors.length === 0, + errors, + }; +} diff --git a/packages/cli-v3/tests/python-dependencies.test.ts b/packages/cli-v3/tests/python-dependencies.test.ts new file mode 100644 index 0000000000..db646cfb09 --- /dev/null +++ b/packages/cli-v3/tests/python-dependencies.test.ts @@ -0,0 +1,210 @@ +import { describe, it, expect } from "vitest"; +import { + parseRequirementsTxt, + generateRequirementsTxt, + validateRequirementsTxt, + type PythonDependency, +} from "../src/build/pythonDependencies.js"; + +describe("Python Dependencies - Requirements.txt parser", () => { + describe("parseRequirementsTxt", () => { + it("parses basic package names", () => { + const content = ` +pydantic +requests +numpy + `.trim(); + + const deps = parseRequirementsTxt(content); + + expect(deps).toHaveLength(3); + expect(deps[0]).toEqual({ name: "pydantic" }); + expect(deps[1]).toEqual({ name: "requests" }); + expect(deps[2]).toEqual({ name: "numpy" }); + }); + + it("parses packages with exact versions", () => { + const content = ` +pydantic==2.0.0 +requests==2.28.0 +numpy==1.24.0 + `.trim(); + + const deps = parseRequirementsTxt(content); + + expect(deps).toHaveLength(3); + expect(deps[0]).toEqual({ name: "pydantic", version: "==2.0.0" }); + expect(deps[1]).toEqual({ name: "requests", version: "==2.28.0" }); + expect(deps[2]).toEqual({ name: "numpy", version: "==1.24.0" }); + }); + + it("parses packages with version ranges", () => { + const content = ` +requests>=2.28.0 +numpy>=1.20.0,<2.0.0 +pydantic~=2.0.0 + `.trim(); + + const deps = parseRequirementsTxt(content); + + expect(deps).toHaveLength(3); + expect(deps[0]).toEqual({ name: "requests", version: ">=2.28.0" }); + expect(deps[1].name).toBe("numpy"); + expect(deps[1].version).toContain(">=1.20.0"); + expect(deps[1].version).toContain("<2.0.0"); + expect(deps[2]).toEqual({ name: "pydantic", version: "~=2.0.0" }); + }); + + it("parses packages with extras", () => { + const content = ` +requests[security,socks]==2.28.0 +pydantic[email]>=2.0.0 + `.trim(); + + const deps = parseRequirementsTxt(content); + + expect(deps).toHaveLength(2); + expect(deps[0]).toEqual({ + name: "requests", + extras: ["security", "socks"], + version: "==2.28.0", + }); + expect(deps[1]).toEqual({ + name: "pydantic", + extras: ["email"], + version: ">=2.0.0", + }); + }); + + it("ignores comments and empty lines", () => { + const content = ` +# This is a comment +pydantic==2.0.0 + +# Another comment +requests>=2.28.0 + + `.trim(); + + const deps = parseRequirementsTxt(content); + + expect(deps).toHaveLength(2); + expect(deps[0]).toEqual({ name: "pydantic", version: "==2.0.0" }); + expect(deps[1]).toEqual({ name: "requests", version: ">=2.28.0" }); + }); + + it("handles mixed operators and extras", () => { + const content = ` +scipy[extra1,extra2]==1.9.0 +matplotlib>=3.0.0,<4.0.0 +pandas + `.trim(); + + const deps = parseRequirementsTxt(content); + + expect(deps).toHaveLength(3); + expect(deps[0]).toEqual({ + name: "scipy", + extras: ["extra1", "extra2"], + version: "==1.9.0", + }); + expect(deps[1].name).toBe("matplotlib"); + expect(deps[1].version).toContain(">=3.0.0"); + expect(deps[1].version).toContain("<4.0.0"); + expect(deps[2]).toEqual({ name: "pandas" }); + }); + + it("handles packages with hyphens and underscores", () => { + const content = ` +flask-cors==3.0.0 +some_package>=1.0.0 + `.trim(); + + const deps = parseRequirementsTxt(content); + + expect(deps).toHaveLength(2); + expect(deps[0]).toEqual({ name: "flask-cors", version: "==3.0.0" }); + expect(deps[1]).toEqual({ name: "some_package", version: ">=1.0.0" }); + }); + }); + + describe("generateRequirementsTxt", () => { + it("generates requirements.txt from dependencies", () => { + const deps: PythonDependency[] = [ + { name: "pydantic", version: "==2.0.0" }, + { name: "requests", version: ">=2.28.0" }, + { name: "numpy" }, + ]; + + const content = generateRequirementsTxt(deps); + + expect(content).toBe("pydantic==2.0.0\nrequests>=2.28.0\nnumpy"); + }); + + it("generates requirements.txt with extras", () => { + const deps: PythonDependency[] = [ + { name: "requests", version: "==2.28.0", extras: ["security", "socks"] }, + { name: "pydantic", extras: ["email", "dotenv"] }, + ]; + + const content = generateRequirementsTxt(deps); + + expect(content).toBe("requests[security,socks]==2.28.0\npydantic[email,dotenv]"); + }); + + it("round-trips parse and generate", () => { + const originalContent = ` +# Production dependencies +pydantic==2.0.0 +requests[security,socks]>=2.28.0 +numpy>=1.20.0 + `.trim(); + + const deps = parseRequirementsTxt(originalContent); + const regeneratedContent = generateRequirementsTxt(deps); + + // Parse again to verify round-trip + const deps2 = parseRequirementsTxt(regeneratedContent); + + expect(deps).toHaveLength(deps2.length); + expect(deps[0]).toEqual(deps2[0]); + expect(deps[1]).toEqual(deps2[1]); + expect(deps[2]).toEqual(deps2[2]); + }); + }); + + describe("validateRequirementsTxt", () => { + it("validates correct requirements.txt", () => { + const content = ` +pydantic==2.0.0 +requests[security]>=2.28.0 +numpy>=1.20.0,<2.0.0 + `.trim(); + + const result = validateRequirementsTxt(content); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it("reports errors with line numbers", () => { + const content = ` +pydantic==2.0.0 +!invalid-package +requests>=2.28.0 + `.trim(); + + const result = validateRequirementsTxt(content); + + expect(result.valid).toBe(false); + expect(result.errors.some((e) => e.includes("Line"))).toBe(true); + }); + + it("handles empty content", () => { + const result = validateRequirementsTxt(""); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + }); +}); diff --git a/packages/core/src/v3/schemas/build.ts b/packages/core/src/v3/schemas/build.ts index ec16f3467e..be6d11a444 100644 --- a/packages/core/src/v3/schemas/build.ts +++ b/packages/core/src/v3/schemas/build.ts @@ -68,6 +68,7 @@ export const BuildManifest = z.object({ exclude: z.array(z.string()).optional(), }) .optional(), + requirementsContent: z.string().optional(), // For Python: requirements.txt content }); export type BuildManifest = z.infer; diff --git a/packages/core/src/v3/schemas/schemas.ts b/packages/core/src/v3/schemas/schemas.ts index 233068c0b7..d848208890 100644 --- a/packages/core/src/v3/schemas/schemas.ts +++ b/packages/core/src/v3/schemas/schemas.ts @@ -199,6 +199,7 @@ export type TaskMetadata = z.infer; export const TaskFile = z.object({ entry: z.string(), out: z.string(), + filePath: z.string().optional(), // For Python: relative path to source file }); export type TaskFile = z.infer; From e71b0de0b50fdd7be78f70729511bac07e2d32dd Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sun, 9 Nov 2025 12:15:13 +0300 Subject: [PATCH 08/10] fix(python): fix IPC and PYTHONPATH for Python build system --- packages/cli-v3/package.json | 3 +- packages/cli-v3/src/build/bundle.ts | 5 +- packages/cli-v3/src/build/pythonBundler.ts | 27 ++++---- .../cli-v3/src/build/pythonDependencies.ts | 16 ++--- packages/cli-v3/src/deploy/buildImage.ts | 9 ++- packages/cli-v3/src/dev/devSupervisor.ts | 18 +++++- .../src/entryPoints/dev-run-controller.ts | 8 +-- .../src/entryPoints/managed/execution.ts | 10 +-- .../python/managed-index-worker.py | 62 +++++++++++++------ .../src/indexing/indexWorkerManifest.ts | 21 ++++++- packages/cli-v3/src/python/pythonProcess.ts | 8 ++- .../cli-v3/src/python/pythonTaskRunner.ts | 10 +-- packages/cli-v3/src/python/stdioIpc.ts | 7 ++- .../cli-v3/tests/fixtures/test-manifest.json | 4 +- packages/cli-v3/tests/python-ipc.test.ts | 8 +-- references/hello-world-python/.gitignore | 1 + references/hello-world-python/package.json | 5 ++ .../hello-world-python/requirements.txt | 3 + .../src/trigger/another_task.py | 10 +++ .../src/trigger/example_task.py | 17 +++++ .../hello-world-python/trigger.config.ts | 6 ++ 21 files changed, 182 insertions(+), 76 deletions(-) create mode 100644 references/hello-world-python/.gitignore create mode 100644 references/hello-world-python/package.json create mode 100644 references/hello-world-python/requirements.txt create mode 100644 references/hello-world-python/src/trigger/another_task.py create mode 100644 references/hello-world-python/src/trigger/example_task.py create mode 100644 references/hello-world-python/trigger.config.ts diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index cc9c931eb2..c83b0762c2 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -72,7 +72,8 @@ "scripts": { "clean": "rimraf dist .tshy .tshy-build .turbo", "typecheck": "tsc -p tsconfig.src.json --noEmit", - "build": "tshy && pnpm run update-version", + "build": "tshy && pnpm run update-version && pnpm run copy-python", + "copy-python": "mkdir -p dist/esm/entryPoints/python && cp src/entryPoints/python/*.py dist/esm/entryPoints/python/", "dev": "tshy --watch", "test": "vitest", "test:e2e": "vitest --run -c ./e2e/vitest.config.ts", diff --git a/packages/cli-v3/src/build/bundle.ts b/packages/cli-v3/src/build/bundle.ts index 1f8a8ee34a..2984f9bd2f 100644 --- a/packages/cli-v3/src/build/bundle.ts +++ b/packages/cli-v3/src/build/bundle.ts @@ -4,7 +4,7 @@ import { BuildManifest, BuildTarget, TaskFile } from "@trigger.dev/core/v3/schem import * as esbuild from "esbuild"; import { createHash } from "node:crypto"; import path, { join, relative, resolve } from "node:path"; -import fs from "fs-extra"; +import { writeFile } from "node:fs/promises"; import { createFile } from "../utilities/fileSystem.js"; import { logger } from "../utilities/logger.js"; import { resolveFileSources } from "../utilities/sourceFiles.js"; @@ -471,14 +471,13 @@ async function bundlePythonWorker(options: BundleOptions): Promise // Create complete BuildManifest const buildManifest = await createBuildManifestFromPythonBundle(bundleResult, { outputDir: destination, - projectDir: cwd, config: resolvedConfig, target, }); // Write manifest to output const manifestPath = join(destination, "build-manifest.json"); - await fs.writeJson(manifestPath, buildManifest, { spaces: 2 }); + await writeFile(manifestPath, JSON.stringify(buildManifest, null, 2)); // Convert to BundleResult const pythonBundleResult: BundleResult = { diff --git a/packages/cli-v3/src/build/pythonBundler.ts b/packages/cli-v3/src/build/pythonBundler.ts index e8c8e978ce..b6514cc843 100644 --- a/packages/cli-v3/src/build/pythonBundler.ts +++ b/packages/cli-v3/src/build/pythonBundler.ts @@ -5,7 +5,7 @@ */ import path from "path"; -import fs from "fs-extra"; +import { readFile, copyFile, mkdir, access } from "node:fs/promises"; import { createHash } from "node:crypto"; import type { BuildManifest, BuildTarget } from "@trigger.dev/core/v3"; import type { ResolvedConfig } from "@trigger.dev/core/v3/build"; @@ -13,6 +13,7 @@ import { logger } from "../utilities/logger.js"; import { parseRequirementsTxt } from "./pythonDependencies.js"; import { VERSION } from "../version.js"; import { CORE_VERSION } from "@trigger.dev/core/v3"; +import { sourceDir } from "../sourceDir.js"; export interface PythonBundleOptions { entryPoints: string[]; // Absolute paths to Python files @@ -48,7 +49,7 @@ export async function bundlePython(options: PythonBundleOptions): Promise e.trim()), - version: version ? `${operator}${version.trim()}` : undefined, - }); + if (name) { + dependencies.push({ + name: name.trim(), + extras: extras?.split(",").map((e) => e.trim()), + version: version ? `${operator}${version.trim()}` : undefined, + }); + } } else { // Try simple package name (reusing the same regex pattern) const simpleMatch = trimmed.match(/^([A-Za-z_][A-Za-z0-9_-]*)/); - if (simpleMatch) { + if (simpleMatch?.[1]) { dependencies.push({ name: simpleMatch[1], }); @@ -76,7 +78,7 @@ export function validateRequirementsTxt(content: string): { valid: boolean; erro const errors: string[] = []; for (let i = 0; i < lines.length; i++) { - const line = lines[i].trim(); + const line = lines[i]?.trim(); // Skip empty lines and comments if (!line || line.startsWith("#")) continue; diff --git a/packages/cli-v3/src/deploy/buildImage.ts b/packages/cli-v3/src/deploy/buildImage.ts index 7aed546862..4843093631 100644 --- a/packages/cli-v3/src/deploy/buildImage.ts +++ b/packages/cli-v3/src/deploy/buildImage.ts @@ -625,11 +625,12 @@ const BASE_IMAGE: Record = { node: "node:21.7.3-bookworm-slim@sha256:dfc05dee209a1d7adf2ef189bd97396daad4e97c6eaa85778d6f75205ba1b0fb", "node-22": "node:22.16.0-bookworm-slim@sha256:048ed02c5fd52e86fda6fbd2f6a76cf0d4492fd6c6fee9e2c463ed5108da0e34", + python: "python:3.12-slim-bookworm@sha256:5dc6f84b5e2d9abc9a5f654ede91c47eec11c2ae3f82db452c5f5c61d5803c8d", }; const DEFAULT_PACKAGES = ["busybox", "ca-certificates", "dumb-init", "git", "openssl"]; -export async function generateContainerfile(options: GenerateContainerfileOptions) { +export async function generateContainerfile(options: GenerateContainerfileOptions): Promise { switch (options.runtime) { case "node": case "node-22": { @@ -638,6 +639,12 @@ export async function generateContainerfile(options: GenerateContainerfileOption case "bun": { return await generateBunContainerfile(options); } + case "python": { + throw new Error("Python Containerfile generation not implemented yet (Task 08)"); + } + default: { + throw new Error(`Unsupported runtime: ${options.runtime}`); + } } } diff --git a/packages/cli-v3/src/dev/devSupervisor.ts b/packages/cli-v3/src/dev/devSupervisor.ts index fdbc3795fb..d84c81e104 100644 --- a/packages/cli-v3/src/dev/devSupervisor.ts +++ b/packages/cli-v3/src/dev/devSupervisor.ts @@ -1,4 +1,5 @@ import { setTimeout as awaitTimeout } from "node:timers/promises"; +import path from "node:path"; import { BuildManifest, CreateBackgroundWorkerRequestBody, @@ -27,6 +28,7 @@ import { resolveLocalEnvVars } from "../utilities/localEnvVars.js"; import type { Metafile } from "esbuild"; import { TaskRunProcessPool } from "./taskRunProcessPool.js"; import { tryCatch } from "@trigger.dev/core/utils"; +import { sourceDir } from "../sourceDir.js"; export type WorkerRuntimeOptions = { name: string | undefined; @@ -454,7 +456,7 @@ class DevSupervisor implements WorkerRuntime { "," ); - return { + const baseEnv = { ...resolveLocalEnvVars( this.options.args.envFile, environmentVariablesResponse.success ? environmentVariablesResponse.data.variables : {} @@ -468,6 +470,20 @@ class DevSupervisor implements WorkerRuntime { }), OTEL_IMPORT_HOOK_INCLUDES, }; + + // Add PYTHONPATH for Python runtime (points to python-sdk for dev mode) + if (this.options.config.runtime === "python") { + // sourceDir = /Users/.../packages/cli-v3/dist/esm/ + // python-sdk = /Users/.../packages/python-sdk/ + const pythonSdkPath = path.join(sourceDir, "../../../python-sdk"); + + return { + ...baseEnv, + PYTHONPATH: pythonSdkPath, + }; + } + + return baseEnv; } async #registerWorker(worker: BackgroundWorker) { diff --git a/packages/cli-v3/src/entryPoints/dev-run-controller.ts b/packages/cli-v3/src/entryPoints/dev-run-controller.ts index 367066bbad..b63be20394 100644 --- a/packages/cli-v3/src/entryPoints/dev-run-controller.ts +++ b/packages/cli-v3/src/entryPoints/dev-run-controller.ts @@ -603,13 +603,7 @@ export class DevRunController { logger.debug("Executing Python task", { taskId: execution.task.id }); const pythonRunner = new PythonTaskRunner(); - const completion = await pythonRunner.executeTask({ - ...execution, - worker: { - runtime: "python", - manifestPath: join(this.opts.worker.build.outputPath, "index.json"), - }, - }); + const completion = await pythonRunner.executeTask(execution); logger.debug("Completed Python run", completion); diff --git a/packages/cli-v3/src/entryPoints/managed/execution.ts b/packages/cli-v3/src/entryPoints/managed/execution.ts index b66f172f4e..a7ea602ed9 100644 --- a/packages/cli-v3/src/entryPoints/managed/execution.ts +++ b/packages/cli-v3/src/entryPoints/managed/execution.ts @@ -611,17 +611,11 @@ export class RunExecution { const taskRunEnv = this.currentTaskRunEnv ?? envVars; // Check if this is a Python task - use PythonTaskRunner instead of TaskRunProcess - if (this.opts.workerManifest.runtime === "python") { + if (this.workerManifest.runtime === "python") { this.sendDebugLog("executing Python task", { taskId: execution.task.id }); const pythonRunner = new PythonTaskRunner(); - const completion = await pythonRunner.executeTask({ - ...execution, - worker: { - runtime: "python", - manifestPath: this.env.TRIGGER_WORKER_MANIFEST_PATH, - }, - }); + const completion = await pythonRunner.executeTask(execution); this.sendDebugLog("completed Python run attempt", { attemptSuccess: completion.ok }); diff --git a/packages/cli-v3/src/entryPoints/python/managed-index-worker.py b/packages/cli-v3/src/entryPoints/python/managed-index-worker.py index f6c0cd39ce..1bbe3ead3e 100755 --- a/packages/cli-v3/src/entryPoints/python/managed-index-worker.py +++ b/packages/cli-v3/src/entryPoints/python/managed-index-worker.py @@ -29,7 +29,7 @@ def load_manifest() -> Dict[str, Any]: """Load build manifest from file or environment""" - manifest_path = os.getenv("TRIGGER_MANIFEST_PATH", "./build-manifest.json") + manifest_path = os.getenv("TRIGGER_BUILD_MANIFEST_PATH", "./build-manifest.json") try: with open(manifest_path, "r") as f: @@ -86,18 +86,23 @@ def collect_task_metadata() -> List[Dict[str, Any]]: # Get task metadata task_meta = task.get_metadata() - # Convert to TaskResource schema - # Note: Convert retry/queue to dicts to handle schema differences - task_resource = TaskResource( - id=task_meta.id, - filePath=task_meta.filePath, - exportName=task_meta.exportName, - retry=task_meta.retry.model_dump() if task_meta.retry else None, - queue=task_meta.queue.model_dump() if task_meta.queue else None, - maxDuration=task_meta.maxDuration, - ) - - tasks.append(task_resource.model_dump()) + # Build task dict with required and optional fields + task_dict = { + "id": task_meta.id, + "filePath": task_meta.filePath, + "exportName": task_meta.exportName, + "entryPoint": task_meta.filePath, # Required by TypeScript schema + } + + # Add optional fields only if they have values + if task_meta.retry: + task_dict["retry"] = task_meta.retry.model_dump() + if task_meta.queue: + task_dict["queue"] = task_meta.queue.model_dump() + if task_meta.maxDuration is not None: + task_dict["maxDuration"] = task_meta.maxDuration + + tasks.append(task_dict) logger.debug(f"Collected task: {task_id}") except Exception as e: logger.error(f"Failed to get metadata for task {task_id}: {e}") @@ -111,10 +116,10 @@ async def main(): # Load manifest manifest = load_manifest() - logger.info(f"Loaded manifest with {len(manifest.get('tasks', []))} task files") + logger.info(f"Loaded manifest with {len(manifest.get('files', []))} task files") # Import all task files - task_files = manifest.get("tasks", []) + task_files = manifest.get("files", []) success_count = 0 for task_file in task_files: @@ -128,10 +133,29 @@ async def main(): tasks = collect_task_metadata() logger.info(f"Found {len(tasks)} tasks") - # Send INDEX_TASKS_COMPLETE message - ipc = StdioIpcConnection() - message = IndexTasksCompleteMessage(tasks=tasks) - await ipc.send(message) + # Build WorkerManifest format (matching TypeScript expectations) + worker_manifest = { + "configPath": manifest.get("configPath", "trigger.config.ts"), + "tasks": tasks, + "incompatiblePackages": [], + "workerEntryPoint": manifest.get("runWorkerEntryPoint", ""), + "runtime": manifest.get("runtime", "python"), + } + + # Send INDEX_COMPLETE message (matching TypeScript schema) + # Note: We send raw JSON to stdout instead of using Pydantic message + # because the TypeScript IPC format is different + index_complete_msg = { + "type": "INDEX_COMPLETE", + "version": "v1", + "payload": { + "manifest": worker_manifest, + "importErrors": [], + } + } + + sys.stdout.write(json.dumps(index_complete_msg) + "\n") + sys.stdout.flush() logger.info("Indexing complete") diff --git a/packages/cli-v3/src/indexing/indexWorkerManifest.ts b/packages/cli-v3/src/indexing/indexWorkerManifest.ts index e4ae72283f..0c5b7c77c3 100644 --- a/packages/cli-v3/src/indexing/indexWorkerManifest.ts +++ b/packages/cli-v3/src/indexing/indexWorkerManifest.ts @@ -108,7 +108,26 @@ export async function indexWorkerManifest({ }); child.stdout?.on("data", (data) => { - handleStdout?.(data.toString()); + const output = data.toString(); + handleStdout?.(output); + + // For Python runtime, parse JSON messages from stdout + if (runtime === "python") { + const lines = output.split("\n").filter((line: string) => line.trim()); + for (const line of lines) { + try { + const parsed = JSON.parse(line); + // Check if this is an IPC message (not a log) + if (parsed.type && parsed.version) { + const message = parseMessageFromCatalog(parsed, indexerToWorkerMessages); + // Trigger the same handler as IPC messages + child.emit("message", message); + } + } catch { + // Not JSON or not a message, ignore (probably a log) + } + } + } }); child.stderr?.on("data", (data) => { diff --git a/packages/cli-v3/src/python/pythonProcess.ts b/packages/cli-v3/src/python/pythonProcess.ts index ee35e82861..a404b219d1 100644 --- a/packages/cli-v3/src/python/pythonProcess.ts +++ b/packages/cli-v3/src/python/pythonProcess.ts @@ -66,19 +66,21 @@ export class PythonProcess { async kill(signal: NodeJS.Signals = "SIGTERM"): Promise { if (!this.process) return; + const process = this.process; + return new Promise((resolve) => { const timeout = setTimeout(() => { logger.warn("Python worker did not exit gracefully, forcing kill"); - this.process?.kill("SIGKILL"); + process.kill("SIGKILL"); resolve(); }, 5000); - this.process.once("exit", () => { + process.once("exit", () => { clearTimeout(timeout); resolve(); }); - this.process.kill(signal); + process.kill(signal); }); } diff --git a/packages/cli-v3/src/python/pythonTaskRunner.ts b/packages/cli-v3/src/python/pythonTaskRunner.ts index 2e6e1acdd3..8ec78823fa 100644 --- a/packages/cli-v3/src/python/pythonTaskRunner.ts +++ b/packages/cli-v3/src/python/pythonTaskRunner.ts @@ -21,12 +21,12 @@ export class PythonTaskRunner { // Determine PYTHONPATH - use env var if set, otherwise use relative path for dev const pythonPath = process.env.TRIGGER_PYTHON_SDK_PATH ? process.env.TRIGGER_PYTHON_SDK_PATH - : path.join(__dirname, "../../../python-sdk"); + : path.join(__dirname, "../../../../python-sdk"); const pythonProcess = new PythonProcess({ workerScript, env: { - TRIGGER_MANIFEST_PATH: execution.worker.manifestPath, + TRIGGER_MANIFEST_PATH: process.env.TRIGGER_WORKER_MANIFEST_PATH || "", // Add SDK path for dev mode (in production, SDK is installed via pip) ...(process.env.TRIGGER_PYTHON_SDK_PATH || process.env.NODE_ENV !== "production" ? { PYTHONPATH: pythonPath } @@ -42,12 +42,13 @@ export class PythonTaskRunner { const result = await new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject(new Error("Task execution timeout")); - }, execution.task.maxDuration ?? 300000); + }, execution.run.maxDuration ?? 300000); ipc.on("TASK_RUN_COMPLETED", (message: any) => { clearTimeout(timeout); resolve({ ok: true, + id: execution.run.id, output: message.completion.output, outputType: message.completion.outputType, usage: message.completion.usage, @@ -58,6 +59,7 @@ export class PythonTaskRunner { clearTimeout(timeout); resolve({ ok: false, + id: execution.run.id, error: message.completion.error, usage: message.completion.usage, }); @@ -88,7 +90,7 @@ export class PythonTaskRunner { id: execution.run.id, payload: JSON.stringify(execution.run.payload), // CRITICAL: Must be JSON string payloadType: execution.run.payloadType, - context: execution.run.context, + traceContext: execution.run.traceContext, tags: execution.run.tags, isTest: execution.run.isTest, }, diff --git a/packages/cli-v3/src/python/stdioIpc.ts b/packages/cli-v3/src/python/stdioIpc.ts index 056355ae25..d7bc342192 100644 --- a/packages/cli-v3/src/python/stdioIpc.ts +++ b/packages/cli-v3/src/python/stdioIpc.ts @@ -54,9 +54,12 @@ const TaskHeartbeatSchema = z.object({ }); const IndexCompleteSchema = z.object({ - type: z.literal("INDEX_TASKS_COMPLETE"), + type: z.literal("INDEX_COMPLETE"), version: z.literal("v1"), - tasks: z.array(z.record(z.any())), + payload: z.object({ + manifest: z.record(z.any()), + importErrors: z.array(z.any()), + }), }); const WorkerMessageSchema = z.discriminatedUnion("type", [ diff --git a/packages/cli-v3/tests/fixtures/test-manifest.json b/packages/cli-v3/tests/fixtures/test-manifest.json index ef11787848..bda606f1ff 100644 --- a/packages/cli-v3/tests/fixtures/test-manifest.json +++ b/packages/cli-v3/tests/fixtures/test-manifest.json @@ -1,7 +1,7 @@ { - "tasks": [ + "files": [ { - "filePath": "packages/cli-v3/tests/python/test-task.py" + "filePath": "/Users/beydogan/dev/trigger.dev/packages/cli-v3/tests/python/test-task.py" } ] } diff --git a/packages/cli-v3/tests/python-ipc.test.ts b/packages/cli-v3/tests/python-ipc.test.ts index d307e29763..eca1847236 100644 --- a/packages/cli-v3/tests/python-ipc.test.ts +++ b/packages/cli-v3/tests/python-ipc.test.ts @@ -16,7 +16,7 @@ describe("Python IPC", () => { const pythonProcess = new PythonProcess({ workerScript: indexWorker, env: { - TRIGGER_MANIFEST_PATH: manifestPath, + TRIGGER_BUILD_MANIFEST_PATH: manifestPath, PYTHONPATH: path.join(__dirname, "../../python-sdk"), }, }); @@ -26,7 +26,7 @@ describe("Python IPC", () => { const result = await new Promise((resolve, reject) => { const timeout = setTimeout(() => reject(new Error("Timeout")), 10000); - ipc.on("INDEX_TASKS_COMPLETE", (message: any) => { + ipc.on("INDEX_COMPLETE", (message: any) => { clearTimeout(timeout); resolve(message); }); @@ -44,8 +44,8 @@ describe("Python IPC", () => { }); }); - expect(result).toHaveProperty("tasks"); - expect((result as any).tasks.length).toBeGreaterThan(0); + expect(result).toHaveProperty("payload"); + expect((result as any).payload.manifest.tasks.length).toBeGreaterThan(0); await pythonProcess.cleanup(); }); diff --git a/references/hello-world-python/.gitignore b/references/hello-world-python/.gitignore new file mode 100644 index 0000000000..7e3509b25a --- /dev/null +++ b/references/hello-world-python/.gitignore @@ -0,0 +1 @@ +.trigger diff --git a/references/hello-world-python/package.json b/references/hello-world-python/package.json new file mode 100644 index 0000000000..ed6edad91b --- /dev/null +++ b/references/hello-world-python/package.json @@ -0,0 +1,5 @@ +{ + "name": "test-python-build", + "version": "1.0.0", + "private": true +} diff --git a/references/hello-world-python/requirements.txt b/references/hello-world-python/requirements.txt new file mode 100644 index 0000000000..26a5df6626 --- /dev/null +++ b/references/hello-world-python/requirements.txt @@ -0,0 +1,3 @@ +pydantic>=2.0.0 +requests[security]>=2.28.0 +httpx==0.24.1 diff --git a/references/hello-world-python/src/trigger/another_task.py b/references/hello-world-python/src/trigger/another_task.py new file mode 100644 index 0000000000..efa43340ae --- /dev/null +++ b/references/hello-world-python/src/trigger/another_task.py @@ -0,0 +1,10 @@ +""" +Another task to verify multiple file discovery +""" + +from trigger_sdk import task + +@task("async-task", retry={"maxAttempts": 3}) +async def async_task(payload): + """Async task example""" + return {"status": "completed", "input": payload} diff --git a/references/hello-world-python/src/trigger/example_task.py b/references/hello-world-python/src/trigger/example_task.py new file mode 100644 index 0000000000..278f84e8ff --- /dev/null +++ b/references/hello-world-python/src/trigger/example_task.py @@ -0,0 +1,17 @@ +""" +Example Python task for testing build system +""" + +from trigger_sdk import task + +@task("hello-task") +async def hello_task(payload): + """A simple test task""" + return {"message": "Hello from Python!", "payload": payload} + + +@task("data-processor", max_duration=60) +def process_data(payload): + """Sync task example""" + data = payload.get("data", "") + return {"result": data.upper()} diff --git a/references/hello-world-python/trigger.config.ts b/references/hello-world-python/trigger.config.ts new file mode 100644 index 0000000000..0a77ee1308 --- /dev/null +++ b/references/hello-world-python/trigger.config.ts @@ -0,0 +1,6 @@ +export default { + project: "proj_rrkpdguyagvsoktglnod", + runtime: "python", + dirs: ["./src/trigger"], + maxDuration: 300, +}; From f47a578f452a171577ebd2c6c5b5b966796f3707 Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sun, 9 Nov 2025 16:34:51 +0300 Subject: [PATCH 09/10] feat(python): migrate Python workers to gRPC with improved error handling --- package.json | 2 + packages/cli-v3/package.json | 2 + packages/cli-v3/src/build/pythonBundler.ts | 6 +- packages/cli-v3/src/dev/backgroundWorker.ts | 50 ++- .../python/managed-index-worker.py | 17 +- .../entryPoints/python/managed-run-worker.py | 56 ++- packages/cli-v3/src/ipc/grpcServer.ts | 338 ++++++++++++++++ packages/cli-v3/src/ipc/protoLoader.ts | 213 ++++++++++ packages/cli-v3/src/python/pythonProcess.ts | 121 +++++- .../cli-v3/src/python/pythonTaskRunner.ts | 132 ++++--- packages/cli-v3/tests/fixtures/test-task.py | 10 + packages/cli-v3/tests/grpc-python.test.ts | 142 +++++++ packages/core/proto/worker.proto | 254 ++++++++++++ packages/python-sdk/.gitignore | 41 ++ packages/python-sdk/pyproject.toml | 2 + packages/python-sdk/trigger_sdk/__init__.py | 4 +- .../trigger_sdk/generated/__init__.py | 5 + .../python-sdk/trigger_sdk/ipc/__init__.py | 6 +- packages/python-sdk/trigger_sdk/ipc/grpc.py | 366 ++++++++++++++++++ packages/python-sdk/trigger_sdk/logger.py | 61 ++- .../trigger_sdk/schemas/messages.py | 31 +- pnpm-lock.yaml | 16 +- .../hello-world-python/requirements.txt | 3 + .../src/trigger/example_task.py | 35 +- 24 files changed, 1800 insertions(+), 113 deletions(-) create mode 100644 packages/cli-v3/src/ipc/grpcServer.ts create mode 100644 packages/cli-v3/src/ipc/protoLoader.ts create mode 100644 packages/cli-v3/tests/fixtures/test-task.py create mode 100644 packages/cli-v3/tests/grpc-python.test.ts create mode 100644 packages/core/proto/worker.proto create mode 100644 packages/python-sdk/.gitignore create mode 100644 packages/python-sdk/trigger_sdk/generated/__init__.py create mode 100644 packages/python-sdk/trigger_sdk/ipc/grpc.py diff --git a/package.json b/package.json index cd799e5eaa..31328afb3f 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,8 @@ "check-exports": "turbo run check-exports", "clean": "turbo run clean", "clean:node_modules": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +", + "clean:python": "find . -name '*.pyc' -delete && find . -name '__pycache__' -type d -exec rm -rf '{}' + 2>/dev/null || true", + "clean:all": "pnpm run clean && pnpm run clean:python", "typecheck": "turbo run typecheck", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", diff --git a/packages/cli-v3/package.json b/packages/cli-v3/package.json index c83b0762c2..494d952142 100644 --- a/packages/cli-v3/package.json +++ b/packages/cli-v3/package.json @@ -84,6 +84,8 @@ "dependencies": { "@clack/prompts": "0.11.0", "@depot/cli": "0.0.1-cli.2.80.0", + "@grpc/grpc-js": "^1.12.4", + "@grpc/proto-loader": "^0.7.15", "@modelcontextprotocol/sdk": "^1.17.0", "@opentelemetry/api": "1.9.0", "@opentelemetry/api-logs": "0.203.0", diff --git a/packages/cli-v3/src/build/pythonBundler.ts b/packages/cli-v3/src/build/pythonBundler.ts index b6514cc843..9053423402 100644 --- a/packages/cli-v3/src/build/pythonBundler.ts +++ b/packages/cli-v3/src/build/pythonBundler.ts @@ -42,7 +42,7 @@ export interface PythonBundleResult { export async function bundlePython(options: PythonBundleOptions): Promise { const { entryPoints, outputDir, projectDir, requirementsFile, config, target } = options; - logger.info("Bundling Python tasks", { + logger.debug("Bundling Python tasks", { entryPoints: entryPoints.length, outputDir, projectDir, @@ -99,7 +99,7 @@ export async function bundlePython(options: PythonBundleOptions): Promise List[Dict[str, Any]]: async def main(): """Main indexing workflow""" - logger.info("Python index worker starting") + logger.debug("Python index worker starting") # Load manifest manifest = load_manifest() - logger.info(f"Loaded manifest with {len(manifest.get('files', []))} task files") + logger.debug(f"Loaded manifest with {len(manifest.get('files', []))} task files") # Import all task files task_files = manifest.get("files", []) @@ -127,11 +126,11 @@ async def main(): if file_path and import_task_file(file_path): success_count += 1 - logger.info(f"Imported {success_count}/{len(task_files)} task files") + logger.debug(f"Imported {success_count}/{len(task_files)} task files") # Collect task metadata tasks = collect_task_metadata() - logger.info(f"Found {len(tasks)} tasks") + logger.debug(f"Found {len(tasks)} tasks") # Build WorkerManifest format (matching TypeScript expectations) worker_manifest = { @@ -142,9 +141,7 @@ async def main(): "runtime": manifest.get("runtime", "python"), } - # Send INDEX_COMPLETE message (matching TypeScript schema) - # Note: We send raw JSON to stdout instead of using Pydantic message - # because the TypeScript IPC format is different + # Send INDEX_COMPLETE message via stdout (for stdio-based communication) index_complete_msg = { "type": "INDEX_COMPLETE", "version": "v1", @@ -157,14 +154,14 @@ async def main(): sys.stdout.write(json.dumps(index_complete_msg) + "\n") sys.stdout.flush() - logger.info("Indexing complete") + logger.debug("Indexing complete") if __name__ == "__main__": try: asyncio.run(main()) except KeyboardInterrupt: - logger.info("Index worker interrupted") + logger.debug("Index worker interrupted") sys.exit(0) except Exception as e: logger.error(f"Index worker failed: {e}", exception=traceback.format_exc()) diff --git a/packages/cli-v3/src/entryPoints/python/managed-run-worker.py b/packages/cli-v3/src/entryPoints/python/managed-run-worker.py index fd08c3dc8c..010ee40f9b 100755 --- a/packages/cli-v3/src/entryPoints/python/managed-run-worker.py +++ b/packages/cli-v3/src/entryPoints/python/managed-run-worker.py @@ -5,12 +5,13 @@ Executes a single Python task. Flow: -1. Receive EXECUTE_TASK_RUN message from stdin -2. Set up execution context (trace context, env vars, etc.) -3. Import task file and get task from registry -4. Execute task with payload -5. Send TASK_RUN_COMPLETED or TASK_RUN_FAILED_TO_RUN message -6. Handle heartbeat and cancellation +1. Connect to coordinator via gRPC +2. Receive EXECUTE_TASK_RUN message +3. Set up execution context (trace context, env vars, etc.) +4. Import task file and get task from registry +5. Execute task with payload +6. Send TASK_RUN_COMPLETED or TASK_RUN_FAILED_TO_RUN message +7. Handle heartbeat and cancellation """ import sys @@ -25,14 +26,14 @@ # Import SDK (assumes it's installed via pip) from trigger_sdk.task import TASK_REGISTRY, Task -from trigger_sdk.ipc import StdioIpcConnection +from trigger_sdk.ipc import GrpcIpcConnection from trigger_sdk.context import TaskContext, set_current_context, clear_current_context from trigger_sdk.logger import logger from trigger_sdk.schemas import ExecuteTaskRunMessage # Global state -ipc: Optional[StdioIpcConnection] = None +ipc: Optional[GrpcIpcConnection] = None current_task: Optional[asyncio.Task] = None cancelled = False @@ -40,11 +41,13 @@ def signal_handler(signum, frame): """Handle termination signals""" global cancelled - logger.warn(f"Received signal {signum}, cancelling task") cancelled = True if current_task and not current_task.done(): + logger.debug(f"Received signal {signum}, cancelling task") current_task.cancel() + else: + logger.debug(f"Received signal {signum}, shutting down") def import_task_file(file_path: str) -> bool: @@ -106,7 +109,7 @@ async def execute_task_run(message: ExecuteTaskRunMessage): logger.error(f"Failed to parse payload JSON: {e}") raise ValueError(f"Invalid payload JSON: {e}") - logger.info(f"Executing task {task_id} from {task_file}") + logger.debug(f"Executing task {task_id} from {task_file}") # Track start time for usage metrics import time @@ -134,7 +137,7 @@ async def execute_task_run(message: ExecuteTaskRunMessage): ) set_current_context(context) - logger.info(f"Starting task execution (attempt {context.attempt.number})") + logger.debug(f"Starting task execution (attempt {context.attempt.number})") # Start heartbeat with run_id parameter heartbeat_task = asyncio.create_task(heartbeat_loop(run_id)) @@ -147,13 +150,17 @@ async def execute_task_run(message: ExecuteTaskRunMessage): # Calculate duration duration_ms = int((time.time() - start_time) * 1000) - logger.info("Task completed successfully") + logger.debug("Task completed successfully") await ipc.send_completed( id=run_id, output=result, usage={"durationMs": duration_ms} ) + # Wait for message to flush before exiting + await ipc.flush() + ipc.stop() + finally: # Stop heartbeat cancelled = True @@ -172,6 +179,10 @@ async def execute_task_run(message: ExecuteTaskRunMessage): usage={"durationMs": duration_ms} ) + # Wait for message to flush before exiting + await ipc.flush() + ipc.stop() + except Exception as e: duration_ms = int((time.time() - start_time) * 1000) logger.error(f"Task execution failed: {e}", exception=traceback.format_exc()) @@ -181,6 +192,10 @@ async def execute_task_run(message: ExecuteTaskRunMessage): usage={"durationMs": duration_ms} ) + # Wait for message to flush before exiting + await ipc.flush() + ipc.stop() + finally: clear_current_context() current_task = None @@ -189,7 +204,7 @@ async def execute_task_run(message: ExecuteTaskRunMessage): async def handle_cancel(message): """Handle cancellation message""" global current_task - logger.info("Received CANCEL message") + logger.debug("Received CANCEL message") signal_handler(signal.SIGTERM, None) @@ -197,14 +212,17 @@ async def main(): """Main run worker loop""" global ipc, cancelled - logger.info("Python run worker starting") + logger.debug("Python run worker starting") # Set up signal handlers signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) - # Create IPC connection - ipc = StdioIpcConnection() + # Create gRPC IPC connection + ipc = GrpcIpcConnection() + + # Configure logger to use gRPC + logger.set_ipc_connection(ipc) # Register message handlers ipc.on("EXECUTE_TASK_RUN", execute_task_run) @@ -214,19 +232,19 @@ async def main(): try: await ipc.start_listening() except asyncio.CancelledError: - logger.info("Run worker cancelled") + logger.debug("Run worker cancelled") except Exception as e: logger.error(f"Run worker failed: {e}", exception=traceback.format_exc()) sys.exit(1) - logger.info("Run worker stopped") + logger.debug("Run worker stopped") if __name__ == "__main__": try: asyncio.run(main()) except KeyboardInterrupt: - logger.info("Run worker interrupted") + logger.debug("Run worker interrupted") sys.exit(0) except Exception as e: logger.error(f"Run worker failed: {e}", exception=traceback.format_exc()) diff --git a/packages/cli-v3/src/ipc/grpcServer.ts b/packages/cli-v3/src/ipc/grpcServer.ts new file mode 100644 index 0000000000..37d034afe6 --- /dev/null +++ b/packages/cli-v3/src/ipc/grpcServer.ts @@ -0,0 +1,338 @@ +/** + * gRPC Server for worker-coordinator communication. + * + * Runs in the coordinator (TaskRunProcess) and accepts connections + * from worker processes via Unix domain sockets or TCP. + */ + +import * as grpc from '@grpc/grpc-js'; +import { EventEmitter } from 'events'; +import { workerProto, WorkerMessage, CoordinatorMessage, LogLevel } from './protoLoader.js'; +import { logger } from '../utilities/logger.js'; +import os from 'os'; +import path from 'path'; +import fs from 'fs'; + +export interface GrpcServerOptions { + /** + * Transport type: 'unix' for Unix domain sockets, 'tcp' for TCP localhost + */ + transport?: 'unix' | 'tcp'; + + /** + * For TCP: port number (0 = random available port) + */ + tcpPort?: number; + + /** + * For Unix sockets: custom socket path + */ + socketPath?: string; + + /** + * Runner ID (used for socket naming in managed mode) + */ + runnerId?: string; +} + +export class GrpcWorkerServer extends EventEmitter { + private server: grpc.Server; + private address: string | null = null; + private transport: 'unix' | 'tcp'; + private socketPath?: string; + private streams: Map> = new Map(); + + constructor(private options: GrpcServerOptions = {}) { + super(); + + // Auto-detect transport: managed mode uses unix sockets, dev mode uses TCP + this.transport = options.transport ?? (options.runnerId ? 'unix' : 'tcp'); + + this.server = new grpc.Server(); + this.setupService(); + } + + private setupService() { + this.server.addService(workerProto.WorkerService.service, { + Connect: this.handleConnect.bind(this), + }); + } + + private handleConnect( + stream: grpc.ServerDuplexStream + ) { + const connectionId = `conn-${Date.now()}-${Math.random().toString(36).substring(7)}`; + + logger.debug('Worker connected to gRPC server', { connectionId }); + + this.streams.set(connectionId, stream); + this.emit('connection', connectionId, stream); + + // Handle incoming messages from worker + stream.on('data', (message: WorkerMessage) => { + try { + this.handleWorkerMessage(connectionId, message); + } catch (error) { + logger.error('Error handling worker message', { + error: error instanceof Error ? error.message : String(error), + connectionId, + }); + } + }); + + stream.on('end', () => { + logger.debug('Worker stream ended', { connectionId }); + this.streams.delete(connectionId); + this.emit('disconnect', connectionId); + stream.end(); + }); + + stream.on('error', (error) => { + logger.error('Worker stream error', { + error: error.message, + connectionId, + }); + this.streams.delete(connectionId); + this.emit('error', error, connectionId); + }); + } + + private handleWorkerMessage(connectionId: string, message: WorkerMessage) { + // Protobuf oneof fields are at the top level + if (message.task_run_completed) { + logger.debug('Received TASK_RUN_COMPLETED', { + connectionId, + id: message.task_run_completed.completion.id, + }); + this.emit('TASK_RUN_COMPLETED', message.task_run_completed, connectionId); + } else if (message.task_run_failed) { + logger.debug('Received TASK_RUN_FAILED', { + connectionId, + id: message.task_run_failed.completion.id, + }); + this.emit('TASK_RUN_FAILED_TO_RUN', message.task_run_failed, connectionId); + } else if (message.task_heartbeat) { + logger.debug('Received TASK_HEARTBEAT', { + connectionId, + id: message.task_heartbeat.id, + }); + this.emit('TASK_HEARTBEAT', message.task_heartbeat, connectionId); + } else if (message.index_tasks_complete) { + logger.debug('Received INDEX_TASKS_COMPLETE', { + connectionId, + taskCount: message.index_tasks_complete.tasks.length, + }); + this.emit('INDEX_TASKS_COMPLETE', message.index_tasks_complete, connectionId); + } else if (message.log) { + // Handle log messages from worker + this.handleLogMessage(message.log, connectionId); + } else { + logger.warn('Unknown worker message type', { connectionId, message }); + } + } + + private handleLogMessage(logMessage: any, connectionId: string) { + const logData = { + message: logMessage.message, + logger: logMessage.logger, + timestamp: logMessage.timestamp, + connectionId, + ...(logMessage.exception && { exception: logMessage.exception }), + }; + + // Route to appropriate log level + switch (logMessage.level) { + case LogLevel.DEBUG: + logger.debug('Python worker', logData); + break; + case LogLevel.INFO: + logger.info('Python worker', logData); + break; + case LogLevel.WARN: + logger.warn('Python worker', logData); + break; + case LogLevel.ERROR: + logger.error('Python worker', logData); + if (logMessage.exception) { + console.error(`[Python] ${logMessage.message}\n${logMessage.exception}`); + } + break; + default: + logger.info('Python worker', logData); + } + } + + /** + * Send a message to a specific worker connection + */ + sendToWorker(connectionId: string, message: CoordinatorMessage): boolean { + const stream = this.streams.get(connectionId); + if (!stream) { + logger.warn('Attempted to send to non-existent connection', { connectionId }); + return false; + } + + try { + stream.write(message); + return true; + } catch (error) { + logger.error('Error sending message to worker', { + error: error instanceof Error ? error.message : String(error), + connectionId, + }); + return false; + } + } + + /** + * Send a message to all connected workers + */ + broadcast(message: CoordinatorMessage) { + for (const [connectionId, stream] of this.streams) { + try { + stream.write(message); + } catch (error) { + logger.error('Error broadcasting to worker', { + error: error instanceof Error ? error.message : String(error), + connectionId, + }); + } + } + } + + /** + * Start the gRPC server + */ + async start(): Promise { + if (this.address) { + throw new Error('Server already started'); + } + + if (this.transport === 'unix') { + this.address = await this.startUnixSocket(); + } else { + this.address = await this.startTcp(); + } + + logger.debug('gRPC server started', { + transport: this.transport, + address: this.address, + }); + + return this.address; + } + + private async startUnixSocket(): Promise { + // Generate socket path + this.socketPath = this.options.socketPath ?? path.join( + os.tmpdir(), + `trigger-grpc-${this.options.runnerId || process.pid}.sock` + ); + + // Clean up existing socket if it exists + if (fs.existsSync(this.socketPath)) { + logger.debug('Cleaning up existing socket', { path: this.socketPath }); + fs.unlinkSync(this.socketPath); + } + + return new Promise((resolve, reject) => { + this.server.bindAsync( + `unix:${this.socketPath}`, + grpc.ServerCredentials.createInsecure(), + (error, port) => { + if (error) { + reject(error); + } else { + resolve(`unix:${this.socketPath}`); + } + } + ); + }); + } + + private async startTcp(): Promise { + const port = this.options.tcpPort ?? 0; // 0 = random available port + + return new Promise((resolve, reject) => { + this.server.bindAsync( + `localhost:${port}`, + grpc.ServerCredentials.createInsecure(), + (error, boundPort) => { + if (error) { + reject(error); + } else { + resolve(`localhost:${boundPort}`); + } + } + ); + }); + } + + /** + * Stop the gRPC server + */ + async stop(): Promise { + return new Promise((resolve) => { + this.server.tryShutdown(() => { + // Clean up Unix socket if it exists + if (this.socketPath && fs.existsSync(this.socketPath)) { + try { + fs.unlinkSync(this.socketPath); + } catch (error) { + logger.warn('Failed to clean up socket file', { + path: this.socketPath, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + this.streams.clear(); + this.address = null; + resolve(); + }); + }); + } + + /** + * Force shutdown the server + */ + forceShutdown(): void { + this.server.forceShutdown(); + + // Clean up Unix socket + if (this.socketPath && fs.existsSync(this.socketPath)) { + try { + fs.unlinkSync(this.socketPath); + } catch (error) { + logger.warn('Failed to clean up socket file during force shutdown', { + path: this.socketPath, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + this.streams.clear(); + this.address = null; + } + + /** + * Get the server address + */ + getAddress(): string | null { + return this.address; + } + + /** + * Get number of connected workers + */ + getConnectionCount(): number { + return this.streams.size; + } + + /** + * Get all connection IDs + */ + getConnectionIds(): string[] { + return Array.from(this.streams.keys()); + } +} diff --git a/packages/cli-v3/src/ipc/protoLoader.ts b/packages/cli-v3/src/ipc/protoLoader.ts new file mode 100644 index 0000000000..4da6d88670 --- /dev/null +++ b/packages/cli-v3/src/ipc/protoLoader.ts @@ -0,0 +1,213 @@ +/** + * Proto loader for gRPC definitions. + * + * Loads worker.proto file and provides typed gRPC service definitions. + */ + +import * as grpc from '@grpc/grpc-js'; +import * as protoLoader from '@grpc/proto-loader'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { existsSync } from 'fs'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Path to proto file (relative to this file) +// In dist/esm/ipc/, we need to go up 4 levels to reach packages/, then into core/proto/ +// This assumes the standard monorepo structure: +// packages/cli-v3/dist/esm/ipc/protoLoader.js +// packages/core/proto/worker.proto +const PROTO_PATH = join(__dirname, '../../../../core/proto/worker.proto'); + +// Validate proto file exists +if (!existsSync(PROTO_PATH)) { + throw new Error( + `Proto file not found at ${PROTO_PATH}. ` + + `This likely means the directory structure has changed. ` + + `Expected path: packages/core/proto/worker.proto relative to packages/cli-v3/dist/esm/ipc/` + ); +} + +// Proto loader options +const packageDefinition = protoLoader.loadSync(PROTO_PATH, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, +}); + +// Load proto definition +const protoDescriptor = grpc.loadPackageDefinition(packageDefinition) as any; + +// Export the service definition +export const workerProto = protoDescriptor.trigger.worker.v1; + +// Type definitions for messages (matching Python Pydantic schemas) +export interface TaskInfo { + id: string; + file_path: string; +} + +export interface RunInfo { + id: string; + payload: string; // JSON-serialized + payload_type: string; + tags: string[]; + is_test: boolean; +} + +export interface AttemptInfo { + id: string; + number: number; + started_at: string; +} + +export interface TaskRunExecution { + task: TaskInfo; + run: RunInfo; + attempt: AttemptInfo; + batch?: { id: string }; + queue?: { id: string; name: string }; + organization?: { id: string; slug: string; name: string }; + project?: { id: string; ref: string; slug: string; name: string }; + environment?: { id: string; slug: string; type: string }; + deployment?: { id: string; short_code: string; version: string }; +} + +export interface TaskRunExecutionUsage { + duration_ms: number; +} + +export interface TaskRunBuiltInError { + name: string; + message: string; + stack_trace: string; +} + +export interface TaskRunInternalError { + code: string; + message: string; + stack_trace: string; +} + +export interface TaskRunStringError { + raw: string; +} + +export interface TaskRunError { + error?: { + built_in_error?: TaskRunBuiltInError; + internal_error?: TaskRunInternalError; + string_error?: TaskRunStringError; + }; +} + +export interface TaskRunSuccessfulExecutionResult { + id: string; + output?: string; + output_type: string; + usage?: TaskRunExecutionUsage; + task_identifier?: string; +} + +export interface TaskRunFailedExecutionResult { + id: string; + error: TaskRunError; + retry?: { timestamp: number; delay: number }; + skipped_retrying?: boolean; + usage?: TaskRunExecutionUsage; + task_identifier?: string; +} + +// Worker → Coordinator Messages +export interface TaskRunCompletedMessage { + type: string; + version: string; + completion: TaskRunSuccessfulExecutionResult; +} + +export interface TaskRunFailedMessage { + type: string; + version: string; + completion: TaskRunFailedExecutionResult; +} + +export interface TaskHeartbeatMessage { + type: string; + version: string; + id: string; +} + +export interface TaskMetadata { + fields: Record; +} + +export interface IndexTasksCompleteMessage { + type: string; + version: string; + tasks: TaskMetadata[]; +} + +export enum LogLevel { + DEBUG = 0, + INFO = 1, + WARN = 2, + ERROR = 3, +} + +export interface LogMessage { + type: string; + version: string; + level: LogLevel; + message: string; + logger: string; + timestamp: string; + exception?: string; +} + +export interface WorkerMessage { + task_run_completed?: TaskRunCompletedMessage; + task_run_failed?: TaskRunFailedMessage; + task_heartbeat?: TaskHeartbeatMessage; + index_tasks_complete?: IndexTasksCompleteMessage; + log?: LogMessage; +} + +// Coordinator → Worker Messages +export interface ExecuteTaskRunMessage { + type: string; + version: string; + execution: TaskRunExecution; +} + +export interface CancelMessage { + type: string; + version: string; +} + +export interface FlushMessage { + type: string; + version: string; +} + +export interface CoordinatorMessage { + execute_task_run?: ExecuteTaskRunMessage; + cancel?: CancelMessage; + flush?: FlushMessage; +} + +// Service types +export type WorkerServiceClient = grpc.Client & { + Connect: grpc.ClientDuplexStream; +}; + +export type WorkerServiceServer = grpc.Server & { + addService( + service: typeof workerProto.WorkerService.service, + implementation: { + Connect: grpc.handleBidiStreamingCall; + } + ): void; +}; diff --git a/packages/cli-v3/src/python/pythonProcess.ts b/packages/cli-v3/src/python/pythonProcess.ts index a404b219d1..a218896083 100644 --- a/packages/cli-v3/src/python/pythonProcess.ts +++ b/packages/cli-v3/src/python/pythonProcess.ts @@ -1,12 +1,12 @@ /** * Python worker process management. * - * Spawns Python workers and manages their lifecycle. + * Spawns Python workers and manages their lifecycle using gRPC. */ import { spawn, ChildProcess } from "child_process"; import { BuildRuntime } from "@trigger.dev/core/v3"; -import { StdioIpcConnection } from "./stdioIpc.js"; +import { GrpcWorkerServer } from "../ipc/grpcServer.js"; import { logger } from "../utilities/logger.js"; import { execPathForRuntime } from "@trigger.dev/core/v3/build"; @@ -19,17 +19,27 @@ export interface PythonProcessOptions { export class PythonProcess { private process: ChildProcess | undefined; - private ipc: StdioIpcConnection | undefined; + private grpcServer: GrpcWorkerServer | undefined; constructor(private options: PythonProcessOptions) {} - async start(): Promise { + async start(): Promise { const pythonBinary = execPathForRuntime(this.options.runtime ?? "python"); + // Start gRPC server + this.grpcServer = new GrpcWorkerServer({ + transport: "tcp", // Use TCP for dev mode (easier debugging) + tcpPort: 0, // Random available port + }); + + const grpcAddress = await this.grpcServer.start(); + logger.debug("Started gRPC server for Python worker", { address: grpcAddress }); + logger.debug("Starting Python worker process", { binary: pythonBinary, script: this.options.workerScript, cwd: this.options.cwd, + grpcAddress, }); this.process = spawn( @@ -43,24 +53,107 @@ export class PythonProcess { env: { ...process.env, ...this.options.env, - // Ensure unbuffered output PYTHONUNBUFFERED: "1", + TRIGGER_GRPC_ADDRESS: grpcAddress, }, stdio: ["pipe", "pipe", "pipe"], } ); - this.ipc = new StdioIpcConnection({ - process: this.process, - handleStderr: true, + // Handle spawn errors (e.g., Python binary not found, permission denied) + this.process.on("error", (error) => { + logger.error("Failed to spawn Python worker process", { + error: error.message, + binary: pythonBinary, + script: this.options.workerScript, + }); + // Common issues and helpful messages + if (error.message.includes("ENOENT")) { + logger.error( + `Python binary not found at '${pythonBinary}'. ` + + `Please ensure Python is installed and in your PATH, or set PYTHON_PATH environment variable.` + ); + } else if (error.message.includes("EACCES")) { + logger.error( + `Permission denied executing '${pythonBinary}'. ` + + `Please check file permissions.` + ); + } + }); + + // Forward stderr logs from Python process + this.process.stderr?.on("data", (data) => { + const output = data.toString().trim(); + if (!output) return; + + // Try to parse as structured JSON log + const lines = output.split("\n"); + for (const line of lines) { + if (!line.trim()) continue; + + try { + const logEntry = JSON.parse(line); + if (logEntry.level && logEntry.message) { + // Handle structured log with appropriate level + const level = logEntry.level.toLowerCase(); + const logData = { + message: logEntry.message, + ...(logEntry.logger && { logger: logEntry.logger }), + ...(logEntry.exception && { exception: logEntry.exception }), + }; + + switch (level) { + case "debug": + logger.debug("Python worker", logData); + break; + case "info": + logger.info("Python worker", logData); + break; + case "warn": + case "warning": + logger.warn("Python worker", logData); + break; + case "error": + case "critical": + logger.error("Python worker", logData); + if (logEntry.exception) { + console.error(`[Python] ${logEntry.message}\n${logEntry.exception}`); + } + break; + default: + logger.info("Python worker", logData); + } + } else { + // JSON but not structured log format + logger.info("Python worker", { output: line }); + } + } catch { + // Not JSON, treat as regular stderr + logger.error("Python worker stderr", { output: line }); + console.error(`[Python stderr] ${line}`); + } + } + }); + + // Forward stdout logs from Python process + this.process.stdout?.on("data", (data) => { + const output = data.toString().trim(); + if (output) { + logger.info("Python worker stdout", { output }); + console.log(`[Python stdout] ${output}`); + } }); - // Forward logs - this.ipc.on("log", (logData) => { - logger.debug("Python worker log", logData); + // Handle process exit + this.process.on("exit", (code, signal) => { + if (code === 0) { + logger.debug("Python worker exited", { code, signal }); + } else { + logger.error("Python worker exited", { code, signal }); + } }); - return this.ipc; + return this.grpcServer; } async kill(signal: NodeJS.Signals = "SIGTERM"): Promise { @@ -85,8 +178,10 @@ export class PythonProcess { } async cleanup(): Promise { - this.ipc?.close(); await this.kill(); + if (this.grpcServer) { + await this.grpcServer.stop(); + } } get pid(): number | undefined { diff --git a/packages/cli-v3/src/python/pythonTaskRunner.ts b/packages/cli-v3/src/python/pythonTaskRunner.ts index 8ec78823fa..dbf627332c 100644 --- a/packages/cli-v3/src/python/pythonTaskRunner.ts +++ b/packages/cli-v3/src/python/pythonTaskRunner.ts @@ -14,6 +14,66 @@ import { logger } from "../utilities/logger.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); +/** + * Convert protobuf completion message (snake_case) to TypeScript result (camelCase) + */ +function protoCompletionToResult( + completion: any, + runId: string, + isSuccess: boolean +): TaskRunExecutionResult { + if (isSuccess) { + return { + ok: true, + id: runId, + output: completion.output, + outputType: completion.output_type || "application/json", + usage: { + durationMs: parseInt(completion.usage?.duration_ms || "0"), + }, + }; + } else { + return { + ok: false, + id: runId, + error: completion.error, + usage: { + durationMs: parseInt(completion.usage?.duration_ms || "0"), + }, + }; + } +} + +/** + * Convert TypeScript execution (camelCase) to protobuf message (snake_case) + */ +function executionToProtoMessage(execution: TaskRunExecution) { + return { + execute_task_run: { + type: "EXECUTE_TASK_RUN", + version: "v1", + execution: { + task: { + id: execution.task.id, + file_path: execution.task.filePath, + }, + run: { + id: execution.run.id, + payload: JSON.stringify(execution.run.payload), // CRITICAL: Must be JSON string + payload_type: execution.run.payloadType || "application/json", + tags: execution.run.tags || [], + is_test: execution.run.isTest || false, + }, + attempt: { + id: String(execution.attempt.id), + number: execution.attempt.number, + started_at: execution.attempt.startedAt?.toISOString() || new Date().toISOString(), + }, + }, + }, + }; +} + export class PythonTaskRunner { async executeTask(execution: TaskRunExecution): Promise { const workerScript = path.join(__dirname, "../entryPoints/python/managed-run-worker.py"); @@ -36,7 +96,20 @@ export class PythonTaskRunner { }); try { - const ipc = await pythonProcess.start(); + const grpcServer = await pythonProcess.start(); + + // Wait for worker to connect + const connectionId = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error("Worker failed to connect via gRPC")); + }, 10000); + + grpcServer.once("connection", (connId: string) => { + clearTimeout(timeout); + logger.debug("Python worker connected", { connectionId: connId }); + resolve(connId); + }); + }); // Wait for completion or failure const result = await new Promise((resolve, reject) => { @@ -44,65 +117,22 @@ export class PythonTaskRunner { reject(new Error("Task execution timeout")); }, execution.run.maxDuration ?? 300000); - ipc.on("TASK_RUN_COMPLETED", (message: any) => { + grpcServer.on("TASK_RUN_COMPLETED", (message: any) => { clearTimeout(timeout); - resolve({ - ok: true, - id: execution.run.id, - output: message.completion.output, - outputType: message.completion.outputType, - usage: message.completion.usage, - }); + resolve(protoCompletionToResult(message.completion, execution.run.id, true)); }); - ipc.on("TASK_RUN_FAILED_TO_RUN", (message: any) => { + grpcServer.on("TASK_RUN_FAILED_TO_RUN", (message: any) => { clearTimeout(timeout); - resolve({ - ok: false, - id: execution.run.id, - error: message.completion.error, - usage: message.completion.usage, - }); + resolve(protoCompletionToResult(message.completion, execution.run.id, false)); }); - ipc.on("TASK_HEARTBEAT", () => { + grpcServer.on("TASK_HEARTBEAT", () => { logger.debug("Received heartbeat from Python task"); }); - ipc.on("exit", (code: number | null) => { - clearTimeout(timeout); - if (code !== 0) { - reject(new Error(`Python worker exited with code ${code}`)); - } - }); - - // Send execution message - ipc.send({ - type: "EXECUTE_TASK_RUN", - version: "v1", - execution: { - task: { - id: execution.task.id, - filePath: execution.task.filePath, - exportName: execution.task.exportName, - }, - run: { - id: execution.run.id, - payload: JSON.stringify(execution.run.payload), // CRITICAL: Must be JSON string - payloadType: execution.run.payloadType, - traceContext: execution.run.traceContext, - tags: execution.run.tags, - isTest: execution.run.isTest, - }, - attempt: { - id: execution.attempt.id, - number: execution.attempt.number, - startedAt: execution.attempt.startedAt, - backgroundWorkerId: execution.attempt.backgroundWorkerId, - backgroundWorkerTaskId: execution.attempt.backgroundWorkerTaskId, - }, - }, - }); + // Send execution message via gRPC + grpcServer.sendToWorker(connectionId, executionToProtoMessage(execution)); }); return result; diff --git a/packages/cli-v3/tests/fixtures/test-task.py b/packages/cli-v3/tests/fixtures/test-task.py new file mode 100644 index 0000000000..53fd4e27b8 --- /dev/null +++ b/packages/cli-v3/tests/fixtures/test-task.py @@ -0,0 +1,10 @@ +""" +Simple test task for gRPC testing +""" +from trigger_sdk import task + +@task(id="hello-grpc") +async def hello_task(payload): + """Test task that returns a greeting""" + message = payload.get("message", "World") + return {"greeting": f"Hello {message} from Python via gRPC!"} diff --git a/packages/cli-v3/tests/grpc-python.test.ts b/packages/cli-v3/tests/grpc-python.test.ts new file mode 100644 index 0000000000..9365c68964 --- /dev/null +++ b/packages/cli-v3/tests/grpc-python.test.ts @@ -0,0 +1,142 @@ +/** + * Integration test for gRPC communication between Node.js coordinator and Python worker. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawn, ChildProcess } from 'child_process'; +import { GrpcWorkerServer } from '../src/ipc/grpcServer.js'; +import path from 'path'; + +describe('gRPC Python IPC', () => { + let server: GrpcWorkerServer; + let serverAddress: string; + + beforeAll(async () => { + // Start gRPC server + server = new GrpcWorkerServer({ transport: 'tcp', tcpPort: 0 }); + serverAddress = await server.start(); + console.log(`[Test] gRPC server started at ${serverAddress}`); + }); + + afterAll(async () => { + if (server) { + await server.stop(); + console.log('[Test] gRPC server stopped'); + } + }); + + it('should connect Python worker via gRPC and exchange messages', async () => { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error('Test timeout after 10 seconds')); + }, 10000); + + // Track connection + let connectionId: string | null = null; + + // Handle worker connection + server.on('connection', (connId) => { + console.log(`[Test] Worker connected: ${connId}`); + connectionId = connId; + + // Send EXECUTE_TASK_RUN message + const executeMessage = { + message: { + execute_task_run: { + type: 'EXECUTE_TASK_RUN', + version: 'v1', + execution: { + task: { + id: 'test-task', + file_path: 'test.py', + }, + run: { + id: 'run-123', + payload: JSON.stringify({ message: 'Hello from gRPC!' }), + payload_type: 'application/json', + tags: ['test'], + is_test: true, + }, + attempt: { + id: 'attempt-1', + number: 1, + started_at: new Date().toISOString(), + }, + }, + }, + }, + }; + + server.sendToWorker(connId, executeMessage); + console.log('[Test] Sent EXECUTE_TASK_RUN message'); + }); + + // Handle heartbeat + server.on('TASK_HEARTBEAT', (message, connId) => { + console.log(`[Test] Received TASK_HEARTBEAT from ${connId}`, message); + }); + + // Handle task completion + server.on('TASK_RUN_COMPLETED', (message, connId) => { + console.log(`[Test] Received TASK_RUN_COMPLETED from ${connId}`, message); + + clearTimeout(timeout); + + expect(message.completion).toBeDefined(); + expect(message.completion.id).toBe('run-123'); + + // Clean up and resolve + setTimeout(() => { + pythonProcess?.kill(); + resolve(); + }, 500); + }); + + // Handle task failure + server.on('TASK_RUN_FAILED_TO_RUN', (message, connId) => { + console.log(`[Test] Received TASK_RUN_FAILED from ${connId}`, message); + clearTimeout(timeout); + reject(new Error(`Task failed: ${JSON.stringify(message.completion.error)}`)); + }); + + // Handle disconnection + server.on('disconnect', (connId) => { + console.log(`[Test] Worker disconnected: ${connId}`); + }); + + // Spawn Python worker + const pythonWorkerScript = path.join( + __dirname, + '../dist/esm/entryPoints/python/managed-run-worker.py' + ); + + const pythonProcess: ChildProcess = spawn('python3', [pythonWorkerScript], { + env: { + ...process.env, + TRIGGER_GRPC_ADDRESS: serverAddress, + PYTHONPATH: path.join(__dirname, '../../python-sdk'), + PYTHONUNBUFFERED: '1', + }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + + pythonProcess.stdout?.on('data', (data) => { + console.log(`[Python stdout] ${data.toString()}`); + }); + + pythonProcess.stderr?.on('data', (data) => { + console.log(`[Python stderr] ${data.toString()}`); + }); + + pythonProcess.on('error', (error) => { + console.error('[Test] Python process error:', error); + clearTimeout(timeout); + reject(error); + }); + + pythonProcess.on('exit', (code, signal) => { + console.log(`[Test] Python process exited with code ${code}, signal ${signal}`); + }); + }); + }, 15000); // 15 second timeout for the test +}); diff --git a/packages/core/proto/worker.proto b/packages/core/proto/worker.proto new file mode 100644 index 0000000000..11872ca8f1 --- /dev/null +++ b/packages/core/proto/worker.proto @@ -0,0 +1,254 @@ +syntax = "proto3"; + +package trigger.worker.v1; + +// =========================================================================== +// Common Types (matching trigger_sdk/schemas/common.py) +// =========================================================================== + +message TaskInfo { + string id = 1; + string file_path = 2; +} + +message RunInfo { + string id = 1; + string payload = 2; // JSON-serialized + string payload_type = 3; + repeated string tags = 4; + bool is_test = 5; +} + +message AttemptInfo { + string id = 1; + int32 number = 2; + string started_at = 3; // ISO 8601 timestamp +} + +message OrganizationInfo { + string id = 1; + string slug = 2; + string name = 3; +} + +message ProjectInfo { + string id = 1; + string ref = 2; + string slug = 3; + string name = 4; +} + +enum EnvironmentType { + PRODUCTION = 0; + STAGING = 1; + DEVELOPMENT = 2; + PREVIEW = 3; +} + +message EnvironmentInfo { + string id = 1; + string slug = 2; + EnvironmentType type = 3; +} + +message QueueInfo { + string id = 1; + string name = 2; +} + +message DeploymentInfo { + string id = 1; + string short_code = 2; + string version = 3; +} + +message BatchInfo { + string id = 1; +} + +message TaskRunExecution { + // Essential fields (always present) + TaskInfo task = 1; + RunInfo run = 2; + AttemptInfo attempt = 3; + + // Optional fields for progressive expansion + BatchInfo batch = 4; + QueueInfo queue = 5; + OrganizationInfo organization = 6; + ProjectInfo project = 7; + EnvironmentInfo environment = 8; + DeploymentInfo deployment = 9; +} + +message TaskRunExecutionUsage { + int64 duration_ms = 1; +} + +message TaskRunExecutionRetry { + int64 timestamp = 1; // Unix timestamp + int64 delay = 2; // Delay in milliseconds +} + +// =========================================================================== +// Error Types (matching trigger_sdk/schemas/errors.py) +// =========================================================================== + +enum TaskRunErrorCode { + COULD_NOT_IMPORT_TASK = 0; + TASK_EXECUTION_FAILED = 1; + TASK_RUN_CANCELLED = 2; + MAX_DURATION_EXCEEDED = 3; + TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE = 4; + TASK_INPUT_ERROR = 5; + TASK_OUTPUT_ERROR = 6; + INTERNAL_ERROR = 7; +} + +message TaskRunBuiltInError { + string name = 1; // Exception class name + string message = 2; + string stack_trace = 3; +} + +message TaskRunInternalError { + TaskRunErrorCode code = 1; + string message = 2; + string stack_trace = 3; +} + +message TaskRunStringError { + string raw = 1; +} + +message TaskRunError { + oneof error { + TaskRunBuiltInError built_in_error = 1; + TaskRunInternalError internal_error = 2; + TaskRunStringError string_error = 3; + } +} + +// =========================================================================== +// Execution Results (matching trigger_sdk/schemas/common.py) +// =========================================================================== + +message TaskRunSuccessfulExecutionResult { + string id = 1; // Run ID + string output = 2; // JSON-serialized output (optional) + string output_type = 3; + TaskRunExecutionUsage usage = 4; + string task_identifier = 5; // For backwards compatibility (optional) +} + +message TaskRunFailedExecutionResult { + string id = 1; // Run ID + TaskRunError error = 2; + TaskRunExecutionRetry retry = 3; + bool skipped_retrying = 4; + TaskRunExecutionUsage usage = 5; + string task_identifier = 6; // For backwards compatibility (optional) +} + +// =========================================================================== +// Logging (Worker → Coordinator) +// =========================================================================== + +enum LogLevel { + DEBUG = 0; + INFO = 1; + WARN = 2; + ERROR = 3; +} + +message LogMessage { + string type = 1; // Always "LOG" + string version = 2; // Always "v1" + LogLevel level = 3; + string message = 4; + string logger = 5; // Logger name (e.g., "trigger") + string timestamp = 6; // ISO 8601 timestamp + string exception = 7; // Stack trace (optional, for errors) +} + +// =========================================================================== +// Worker → Coordinator Messages (matching trigger_sdk/schemas/messages.py) +// =========================================================================== + +message TaskRunCompletedMessage { + string type = 1; // Always "TASK_RUN_COMPLETED" + string version = 2; // Always "v1" + TaskRunSuccessfulExecutionResult completion = 3; +} + +message TaskRunFailedMessage { + string type = 1; // Always "TASK_RUN_FAILED_TO_RUN" + string version = 2; // Always "v1" + TaskRunFailedExecutionResult completion = 3; +} + +message TaskHeartbeatMessage { + string type = 1; // Always "TASK_HEARTBEAT" + string version = 2; // Always "v1" + string id = 3; // Run or attempt ID +} + +message TaskMetadata { + // Flexible structure for task catalog + // Using map for JSON-like structure + map fields = 1; +} + +message IndexTasksCompleteMessage { + string type = 1; // Always "INDEX_TASKS_COMPLETE" + string version = 2; // Always "v1" + repeated TaskMetadata tasks = 3; +} + +message WorkerMessage { + oneof message { + TaskRunCompletedMessage task_run_completed = 1; + TaskRunFailedMessage task_run_failed = 2; + TaskHeartbeatMessage task_heartbeat = 3; + IndexTasksCompleteMessage index_tasks_complete = 4; + LogMessage log = 5; + } +} + +// =========================================================================== +// Coordinator → Worker Messages (matching trigger_sdk/schemas/messages.py) +// =========================================================================== + +message ExecuteTaskRunMessage { + string type = 1; // Always "EXECUTE_TASK_RUN" + string version = 2; // Always "v1" + TaskRunExecution execution = 3; +} + +message CancelMessage { + string type = 1; // Always "CANCEL" + string version = 2; // Always "v1" +} + +message FlushMessage { + string type = 1; // Always "FLUSH" + string version = 2; // Always "v1" +} + +message CoordinatorMessage { + oneof message { + ExecuteTaskRunMessage execute_task_run = 1; + CancelMessage cancel = 2; + FlushMessage flush = 3; + } +} + +// =========================================================================== +// Bidirectional Streaming Service +// =========================================================================== + +service WorkerService { + // Bidirectional streaming RPC for worker-coordinator communication + // Worker and coordinator both send and receive messages over same connection + rpc Connect(stream WorkerMessage) returns (stream CoordinatorMessage); +} diff --git a/packages/python-sdk/.gitignore b/packages/python-sdk/.gitignore new file mode 100644 index 0000000000..833ce12e17 --- /dev/null +++ b/packages/python-sdk/.gitignore @@ -0,0 +1,41 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ + +# Type checking +.mypy_cache/ +.pytype/ + +# Generated protobuf files +trigger_sdk/generated/*.py +trigger_sdk/generated/*.pyi +!trigger_sdk/generated/__init__.py diff --git a/packages/python-sdk/pyproject.toml b/packages/python-sdk/pyproject.toml index e55dc9f199..b3f70738f4 100644 --- a/packages/python-sdk/pyproject.toml +++ b/packages/python-sdk/pyproject.toml @@ -10,6 +10,8 @@ requires-python = ">=3.10" dependencies = [ "pydantic>=2.0.0", "typing-extensions>=4.5.0", + "grpcio>=1.68.0", + "grpcio-tools>=1.68.0", ] [project.optional-dependencies] diff --git a/packages/python-sdk/trigger_sdk/__init__.py b/packages/python-sdk/trigger_sdk/__init__.py index e9a61230ce..1a48e9124b 100644 --- a/packages/python-sdk/trigger_sdk/__init__.py +++ b/packages/python-sdk/trigger_sdk/__init__.py @@ -2,7 +2,7 @@ from trigger_sdk.task import task, Task, TASK_REGISTRY from trigger_sdk.types import TaskConfig, RetryConfig, QueueConfig -from trigger_sdk.ipc import IpcConnection, StdioIpcConnection +from trigger_sdk.ipc import IpcConnection, GrpcIpcConnection from trigger_sdk.schemas.messages import WorkerMessage, CoordinatorMessage from trigger_sdk.schemas.common import TaskRunExecution from trigger_sdk.context import TaskContext, get_current_context @@ -20,7 +20,7 @@ "QueueConfig", # IPC layer "IpcConnection", - "StdioIpcConnection", + "GrpcIpcConnection", # Message types "WorkerMessage", "CoordinatorMessage", diff --git a/packages/python-sdk/trigger_sdk/generated/__init__.py b/packages/python-sdk/trigger_sdk/generated/__init__.py new file mode 100644 index 0000000000..44b46bf7f9 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/generated/__init__.py @@ -0,0 +1,5 @@ +"""Generated gRPC protocol buffer code.""" + +from trigger_sdk.generated import worker_pb2, worker_pb2_grpc + +__all__ = ["worker_pb2", "worker_pb2_grpc"] diff --git a/packages/python-sdk/trigger_sdk/ipc/__init__.py b/packages/python-sdk/trigger_sdk/ipc/__init__.py index 9a607f72c6..1c0f731437 100644 --- a/packages/python-sdk/trigger_sdk/ipc/__init__.py +++ b/packages/python-sdk/trigger_sdk/ipc/__init__.py @@ -1,11 +1,11 @@ """ IPC (Inter-Process Communication) layer for Python workers. -Provides transport-agnostic message communication between Python workers +Provides gRPC-based message communication between Python workers and the Node.js coordinator. """ from trigger_sdk.ipc.base import IpcConnection -from trigger_sdk.ipc.stdio import StdioIpcConnection +from trigger_sdk.ipc.grpc import GrpcIpcConnection -__all__ = ["IpcConnection", "StdioIpcConnection"] +__all__ = ["IpcConnection", "GrpcIpcConnection"] diff --git a/packages/python-sdk/trigger_sdk/ipc/grpc.py b/packages/python-sdk/trigger_sdk/ipc/grpc.py new file mode 100644 index 0000000000..73d0e19318 --- /dev/null +++ b/packages/python-sdk/trigger_sdk/ipc/grpc.py @@ -0,0 +1,366 @@ +""" +gRPC-based IPC connection for Python workers. + +Implements the IpcConnection interface using gRPC for communication +with the coordinator. +""" + +import os +import asyncio +import sys +import traceback +import logging +from typing import Any, Callable, Dict, Optional +from concurrent.futures import ThreadPoolExecutor + +import grpc + +from trigger_sdk.ipc.base import IpcConnection +from trigger_sdk.schemas.messages import ( + WorkerMessage, + CoordinatorMessage, + TaskRunCompletedMessage, + TaskRunFailedMessage, + TaskHeartbeatMessage, + IndexTasksCompleteMessage, + LogMessage, + ExecuteTaskRunMessage, + CancelMessage, + FlushMessage, +) +from trigger_sdk.schemas.common import ( + TaskRunSuccessfulExecutionResult, + TaskRunFailedExecutionResult, +) +from trigger_sdk.schemas.errors import TaskRunError, TaskRunInternalError +from trigger_sdk.generated import worker_pb2, worker_pb2_grpc + +# Get logger for debug output +_logger = logging.getLogger(__name__) + + +class GrpcIpcConnection(IpcConnection): + """ + gRPC-based IPC connection. + + Connects to the coordinator's gRPC server via Unix socket or TCP. + Compatible with the same interface as StdioIpcConnection. + """ + + def __init__( + self, + address: Optional[str] = None, + ): + """ + Initialize gRPC IPC connection. + + Args: + address: gRPC server address (unix:/path/to/socket or localhost:port) + If None, reads from TRIGGER_GRPC_ADDRESS env var + """ + super().__init__() + + self.address = address or os.environ.get("TRIGGER_GRPC_ADDRESS") + if not self.address: + raise ValueError( + "gRPC address not provided. Set TRIGGER_GRPC_ADDRESS environment variable " + "or pass address parameter." + ) + + self.channel: Optional[grpc.aio.Channel] = None + self.stub: Optional[worker_pb2_grpc.WorkerServiceStub] = None + self.stream: Optional[grpc.aio.StreamStreamCall] = None + self._send_queue: asyncio.Queue = asyncio.Queue() + self._running = False + self._executor = ThreadPoolExecutor(max_workers=1) + self._handlers: Dict[str, Callable] = {} + + def on(self, message_type: str, handler: Callable) -> None: + """Register a message handler""" + self._handlers[message_type] = handler + + async def _dispatch_message(self, message: CoordinatorMessage): + """Dispatch message to registered handler""" + message_type = message.type + + if message_type in self._handlers: + handler = self._handlers[message_type] + + # Support both sync and async handlers + if asyncio.iscoroutinefunction(handler): + await handler(message) + else: + handler(message) + + async def connect(self): + """Establish gRPC connection to coordinator""" + # Create channel (Unix socket or TCP) + self.channel = grpc.aio.insecure_channel(self.address) + + # Create stub + self.stub = worker_pb2_grpc.WorkerServiceStub(self.channel) + + # Start bidirectional stream + self.stream = self.stub.Connect(self._message_generator()) + + async def _message_generator(self): + """Generator for outgoing messages to coordinator""" + while self._running or not self._send_queue.empty(): + try: + # Get message from queue + pydantic_msg = await asyncio.wait_for( + self._send_queue.get(), + timeout=1.0 + ) + + # Convert Pydantic → Protobuf + proto_msg = self._pydantic_to_proto(pydantic_msg) + + yield proto_msg + + except asyncio.TimeoutError: + # Keep generator alive + continue + except Exception as e: + # Log and continue on errors (connection might be closing) + _logger.debug(f"Error in message generator: {e}") + continue + + def _pydantic_to_proto(self, message: WorkerMessage) -> worker_pb2.WorkerMessage: + """Convert Pydantic message to Protobuf""" + proto_msg = worker_pb2.WorkerMessage() + + if isinstance(message, TaskRunCompletedMessage): + completion = message.completion + proto_msg.task_run_completed.type = message.type + proto_msg.task_run_completed.version = message.version + + result = proto_msg.task_run_completed.completion + result.id = completion["id"] + if completion.get("output"): + result.output = completion["output"] + result.output_type = completion.get("outputType", "application/json") + if completion.get("usage"): + result.usage.duration_ms = completion["usage"].get("durationMs", 0) + if completion.get("taskIdentifier"): + result.task_identifier = completion["taskIdentifier"] + + elif isinstance(message, TaskRunFailedMessage): + completion = message.completion + proto_msg.task_run_failed.type = message.type + proto_msg.task_run_failed.version = message.version + + result = proto_msg.task_run_failed.completion + result.id = completion["id"] + + # Set error + error_data = completion["error"] + if error_data.get("type") == "BUILT_IN_ERROR": + result.error.built_in_error.name = error_data["name"] + result.error.built_in_error.message = error_data["message"] + result.error.built_in_error.stack_trace = error_data["stackTrace"] + elif error_data.get("type") == "INTERNAL_ERROR": + result.error.internal_error.code = error_data["code"] + result.error.internal_error.message = error_data.get("message", "") + result.error.internal_error.stack_trace = error_data.get("stackTrace", "") + elif error_data.get("type") == "STRING_ERROR": + result.error.string_error.raw = error_data["raw"] + + if completion.get("usage"): + result.usage.duration_ms = completion["usage"].get("durationMs", 0) + if completion.get("taskIdentifier"): + result.task_identifier = completion["taskIdentifier"] + + elif isinstance(message, TaskHeartbeatMessage): + proto_msg.task_heartbeat.type = message.type + proto_msg.task_heartbeat.version = message.version + proto_msg.task_heartbeat.id = message.id + + elif isinstance(message, IndexTasksCompleteMessage): + proto_msg.index_tasks_complete.type = message.type + proto_msg.index_tasks_complete.version = message.version + for task in message.tasks: + task_meta = proto_msg.index_tasks_complete.tasks.add() + for key, value in task.items(): + task_meta.fields[key] = str(value) + + elif isinstance(message, LogMessage): + proto_msg.log.type = message.type + proto_msg.log.version = message.version + proto_msg.log.level = int(message.level) + proto_msg.log.message = message.message + proto_msg.log.logger = message.logger + proto_msg.log.timestamp = message.timestamp + if message.exception: + proto_msg.log.exception = message.exception + + return proto_msg + + def _proto_to_pydantic(self, proto_msg: worker_pb2.CoordinatorMessage) -> CoordinatorMessage: + """Convert Protobuf message to Pydantic""" + # Check which oneof field is set + which = proto_msg.WhichOneof("message") + + if which == "execute_task_run": + exec_msg = proto_msg.execute_task_run + execution_dict = self._proto_execution_to_dict(exec_msg.execution) + return ExecuteTaskRunMessage( + type=exec_msg.type, + version=exec_msg.version, + execution=execution_dict + ) + elif which == "cancel": + return CancelMessage( + type=proto_msg.cancel.type, + version=proto_msg.cancel.version + ) + elif which == "flush": + return FlushMessage( + type=proto_msg.flush.type, + version=proto_msg.flush.version + ) + + raise ValueError(f"Unknown coordinator message type: {which}") + + def _proto_execution_to_dict(self, execution: worker_pb2.TaskRunExecution) -> Dict[str, Any]: + """Convert TaskRunExecution protobuf to dict for Pydantic validation""" + result = { + "task": { + "id": execution.task.id, + "filePath": execution.task.file_path, + }, + "run": { + "id": execution.run.id, + "payload": execution.run.payload, + "payloadType": execution.run.payload_type, + "tags": list(execution.run.tags), + "isTest": execution.run.is_test, + }, + "attempt": { + "id": execution.attempt.id, + "number": execution.attempt.number, + "startedAt": execution.attempt.started_at, + }, + } + + # Optional fields + if execution.HasField("batch"): + result["batch"] = {"id": execution.batch.id} + if execution.HasField("queue"): + result["queue"] = {"id": execution.queue.id, "name": execution.queue.name} + if execution.HasField("organization"): + result["organization"] = { + "id": execution.organization.id, + "slug": execution.organization.slug, + "name": execution.organization.name, + } + if execution.HasField("project"): + result["project"] = { + "id": execution.project.id, + "ref": execution.project.ref, + "slug": execution.project.slug, + "name": execution.project.name, + } + if execution.HasField("environment"): + result["environment"] = { + "id": execution.environment.id, + "slug": execution.environment.slug, + "type": worker_pb2.EnvironmentType.Name(execution.environment.type), + } + if execution.HasField("deployment"): + result["deployment"] = { + "id": execution.deployment.id, + "shortCode": execution.deployment.short_code, + "version": execution.deployment.version, + } + + return result + + async def send(self, message: WorkerMessage): + """ + Send a message to the coordinator via gRPC. + """ + if not self._running: + raise RuntimeError("gRPC connection not started. Call start_listening() first.") + + try: + # Add message to send queue + await self._send_queue.put(message) + + except Exception as e: + # Log and fail - connection might be closing + _logger.debug(f"Failed to send message: {e}") + pass + + async def start_listening(self): + """ + Start listening for messages from the coordinator. + + Connects to gRPC server and processes incoming messages. + """ + self._running = True + + try: + # Connect to server + await self.connect() + + # Listen for incoming messages + async for proto_msg in self.stream: + try: + # Convert Protobuf → Pydantic + message = self._proto_to_pydantic(proto_msg) + + # Dispatch to handler + await self._dispatch_message(message) + + except Exception as e: + # Log and continue on message handling errors + _logger.debug(f"Error handling message: {e}") + pass + + except grpc.aio.AioRpcError as e: + # Connection closed - this is expected + _logger.debug(f"gRPC connection closed: {e}") + pass + except Exception as e: + # Fatal error - log and exit + _logger.debug(f"Fatal error in gRPC listener: {e}") + pass + finally: + self._running = False + await self.close() + + async def flush(self, timeout: float = 1.0): + """ + Wait for all pending messages to be sent. + + Args: + timeout: Maximum time to wait in seconds + """ + start_time = asyncio.get_event_loop().time() + while not self._send_queue.empty(): + elapsed = asyncio.get_event_loop().time() - start_time + if elapsed >= timeout: + break + await asyncio.sleep(0.01) + + def stop(self): + """Stop listening for messages""" + self._running = False + + async def close(self): + """Close the gRPC connection""" + self.stop() + + # Close stream + if self.stream: + self.stream.cancel() + self.stream = None + + # Close channel + if self.channel: + await self.channel.close() + self.channel = None + + # Shutdown executor + self._executor.shutdown(wait=False) diff --git a/packages/python-sdk/trigger_sdk/logger.py b/packages/python-sdk/trigger_sdk/logger.py index 707cb81d16..8e65c43858 100644 --- a/packages/python-sdk/trigger_sdk/logger.py +++ b/packages/python-sdk/trigger_sdk/logger.py @@ -2,34 +2,89 @@ import json import sys -from typing import Any +import asyncio +from typing import Any, Optional, TYPE_CHECKING from datetime import datetime, timezone from trigger_sdk.context import get_current_context +if TYPE_CHECKING: + from trigger_sdk.ipc.base import IpcConnection + class TaskLogger: """ Structured logger for task execution. - Logs are sent to stderr with structured metadata. + Logs can be sent via gRPC (when available) or stderr (fallback). """ def __init__(self, name: str = "trigger"): self.name = name + self._ipc_connection: Optional["IpcConnection"] = None + + def set_ipc_connection(self, connection: "IpcConnection") -> None: + """Set IPC connection for sending logs via gRPC""" + self._ipc_connection = connection def _log(self, level: str, message: str, **extra: Any) -> None: """Internal log method with structured data""" + from trigger_sdk.schemas.messages import LogMessage, LogLevel + + timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + exception = extra.pop("exception", None) + + # If gRPC is available and we're in an async context, send via gRPC + if self._ipc_connection: + try: + # Check if connection is still running + if not hasattr(self._ipc_connection, '_running') or not self._ipc_connection._running: + # Connection is closed, fall back to stderr + self._log_to_stderr(level, message, timestamp, exception, **extra) + return + + # Map string level to enum + log_level = LogLevel[level] + + log_msg = LogMessage( + level=log_level, + message=message, + logger=self.name, + timestamp=timestamp, + exception=exception, + ) + + # Try to send async if we're in an event loop + try: + loop = asyncio.get_running_loop() + # Schedule the send as a task + asyncio.create_task(self._ipc_connection.send(log_msg)) + except RuntimeError: + # No event loop running, fall back to stderr + self._log_to_stderr(level, message, timestamp, exception, **extra) + + except Exception: + # If gRPC fails, fall back to stderr silently + self._log_to_stderr(level, message, timestamp, exception, **extra) + else: + # No gRPC connection, use stderr + self._log_to_stderr(level, message, timestamp, exception, **extra) + + def _log_to_stderr(self, level: str, message: str, timestamp: str, exception: Optional[str] = None, **extra: Any) -> None: + """Fallback logging to stderr as JSON""" context = get_current_context() log_data = { - "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "timestamp": timestamp, "level": level, "message": message, "logger": self.name, **extra, } + if exception: + log_data["exception"] = exception + # Add context metadata if available if context: log_data["task"] = { diff --git a/packages/python-sdk/trigger_sdk/schemas/messages.py b/packages/python-sdk/trigger_sdk/schemas/messages.py index 577378922a..5dedd8cdc3 100644 --- a/packages/python-sdk/trigger_sdk/schemas/messages.py +++ b/packages/python-sdk/trigger_sdk/schemas/messages.py @@ -5,8 +5,9 @@ packages/core/src/v3/schemas/messages.ts """ -from typing import Any, Dict, Literal, Union +from typing import Any, Dict, Literal, Union, Optional from pydantic import BaseModel, Field +from enum import IntEnum from trigger_sdk.schemas.common import ( TaskRunSuccessfulExecutionResult, @@ -15,6 +16,18 @@ ) +# ============================================================================ +# Logging +# ============================================================================ + +class LogLevel(IntEnum): + """Log level enum matching proto definition""" + DEBUG = 0 + INFO = 1 + WARN = 2 + ERROR = 3 + + # ============================================================================ # Worker → Coordinator Messages # ============================================================================ @@ -74,12 +87,28 @@ class IndexTasksCompleteMessage(BaseModel): tasks: list[Dict[str, Any]] # List of TaskResource as dicts +class LogMessage(BaseModel): + """ + Log message from worker to coordinator. + + Sent to report log entries from the Python worker. + """ + type: Literal["LOG"] = "LOG" + version: Literal["v1"] = "v1" + level: LogLevel + message: str + logger: str # Logger name (e.g., "trigger") + timestamp: str # ISO 8601 timestamp + exception: Optional[str] = None # Stack trace (for errors) + + # Discriminated union of all worker messages WorkerMessage = Union[ TaskRunCompletedMessage, TaskRunFailedMessage, TaskHeartbeatMessage, IndexTasksCompleteMessage, + LogMessage, ] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 92e835de43..04eaba52f9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1309,6 +1309,12 @@ importers: '@depot/cli': specifier: 0.0.1-cli.2.80.0 version: 0.0.1-cli.2.80.0 + '@grpc/grpc-js': + specifier: ^1.12.4 + version: 1.12.6 + '@grpc/proto-loader': + specifier: ^0.7.15 + version: 0.7.15 '@modelcontextprotocol/sdk': specifier: ^1.17.0 version: 1.17.1(supports-color@10.0.0) @@ -2241,6 +2247,8 @@ importers: specifier: workspace:* version: link:../../packages/cli-v3 + references/hello-world-python: {} + references/init-shell: devDependencies: trigger.dev: @@ -7554,11 +7562,11 @@ packages: resolution: {integrity: sha512-JXUj6PI0oqqzTGvKtzOkxtpsyPRNsrmhh41TtIz/zEB6J+AUiZZ0dxWzcMwO9Ns5rmSPuMdghlTbUuqIM48d3Q==} engines: {node: '>=12.10.0'} dependencies: - '@grpc/proto-loader': 0.7.13 + '@grpc/proto-loader': 0.7.15 '@js-sdsl/ordered-map': 4.4.2 - /@grpc/proto-loader@0.7.13: - resolution: {integrity: sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==} + /@grpc/proto-loader@0.7.15: + resolution: {integrity: sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==} engines: {node: '>=6'} hasBin: true dependencies: @@ -21854,7 +21862,7 @@ packages: dependencies: '@balena/dockerignore': 1.0.2 '@grpc/grpc-js': 1.12.6 - '@grpc/proto-loader': 0.7.13 + '@grpc/proto-loader': 0.7.15 docker-modem: 5.0.6 protobufjs: 7.3.2 tar-fs: 2.1.3 diff --git a/references/hello-world-python/requirements.txt b/references/hello-world-python/requirements.txt index 26a5df6626..29d083283c 100644 --- a/references/hello-world-python/requirements.txt +++ b/references/hello-world-python/requirements.txt @@ -1,3 +1,6 @@ pydantic>=2.0.0 requests[security]>=2.28.0 httpx==0.24.1 +grpcio>=1.68.0 +grpcio-tools>=1.68.0 +protobuf>=6.31.1 diff --git a/references/hello-world-python/src/trigger/example_task.py b/references/hello-world-python/src/trigger/example_task.py index 278f84e8ff..15b8ddcd21 100644 --- a/references/hello-world-python/src/trigger/example_task.py +++ b/references/hello-world-python/src/trigger/example_task.py @@ -2,9 +2,10 @@ Example Python task for testing build system """ -from trigger_sdk import task +import asyncio +from trigger_sdk import task, logger -@task("hello-task") +@task("hello-task-v2") async def hello_task(payload): """A simple test task""" return {"message": "Hello from Python!", "payload": payload} @@ -15,3 +16,33 @@ def process_data(payload): """Sync task example""" data = payload.get("data", "") return {"result": data.upper()} + + +@task("long-running-task", max_duration=300) +async def long_running_task(payload): + """A long-running task with multiple logging statementsx""" + logger.info("Starting long-running task", step="initialization") + + # Simulate some initial processing + await asyncio.sleep(2) + logger.debug("Completed initialization phase") + + # Process data in stages + stages = ["data-collection", "data-processing", "data-transformation", "data-validation"] + + for i, stage in enumerate(stages, 1): + logger.info(f"Processing stage {i}/{len(stages)}: {stage}", stage=stage, progress=f"{i}/{len(stages)}") + await asyncio.sleep(3) + logger.debug(f"Completed {stage}", stage=stage) + + # Simulate some final work + logger.info("Finalizing results", step="finalization") + await asyncio.sleep(2) + + logger.info("Task completed successfully", total_duration="~17s") + + return { + "status": "completed", + "stages_processed": len(stages), + "payload": payload + } From b25521c5baab440d2e95c7df302491065d4f6548 Mon Sep 17 00:00:00 2001 From: Mehmet Beydogan Date: Sun, 9 Nov 2025 19:54:11 +0300 Subject: [PATCH 10/10] fix(python): fix max_duration conversion and disable bytecode cache in dev --- package.json | 1 + .../src/entryPoints/python/managed-index-worker.py | 2 ++ packages/cli-v3/src/indexing/indexWorkerManifest.ts | 1 + packages/cli-v3/src/ipc/grpcServer.ts | 9 +++++++-- packages/cli-v3/src/python/pythonProcess.ts | 1 + packages/cli-v3/src/python/pythonTaskRunner.ts | 10 +++++++++- packages/python-sdk/trigger_sdk/task.py | 5 ++++- packages/python-sdk/trigger_sdk/types.py | 2 +- 8 files changed, 26 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 31328afb3f..42005c35df 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "clean:node_modules": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +", "clean:python": "find . -name '*.pyc' -delete && find . -name '__pycache__' -type d -exec rm -rf '{}' + 2>/dev/null || true", "clean:all": "pnpm run clean && pnpm run clean:python", + "rebuild:cli": "pnpm run clean:python && pnpm run build --filter trigger.dev", "typecheck": "turbo run typecheck", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", diff --git a/packages/cli-v3/src/entryPoints/python/managed-index-worker.py b/packages/cli-v3/src/entryPoints/python/managed-index-worker.py index 037ebb4ff4..35e0cc39f1 100755 --- a/packages/cli-v3/src/entryPoints/python/managed-index-worker.py +++ b/packages/cli-v3/src/entryPoints/python/managed-index-worker.py @@ -99,7 +99,9 @@ def collect_task_metadata() -> List[Dict[str, Any]]: if task_meta.queue: task_dict["queue"] = task_meta.queue.model_dump() if task_meta.maxDuration is not None: + # Already in milliseconds from task decorator task_dict["maxDuration"] = task_meta.maxDuration + logger.debug(f"Task {task_id} maxDuration: {task_meta.maxDuration}ms") tasks.append(task_dict) logger.debug(f"Collected task: {task_id}") diff --git a/packages/cli-v3/src/indexing/indexWorkerManifest.ts b/packages/cli-v3/src/indexing/indexWorkerManifest.ts index 0c5b7c77c3..1a7899fdd9 100644 --- a/packages/cli-v3/src/indexing/indexWorkerManifest.ts +++ b/packages/cli-v3/src/indexing/indexWorkerManifest.ts @@ -50,6 +50,7 @@ export async function indexWorkerManifest({ TRIGGER_BUILD_MANIFEST_PATH: buildManifestPath, NODE_OPTIONS: nodeOptions, TRIGGER_INDEXING: "1", + PYTHONDONTWRITEBYTECODE: "1", // Disable .pyc files in dev to avoid stale cache }, execPath: execPathForRuntime(runtime), }); diff --git a/packages/cli-v3/src/ipc/grpcServer.ts b/packages/cli-v3/src/ipc/grpcServer.ts index 37d034afe6..f1f44b9fee 100644 --- a/packages/cli-v3/src/ipc/grpcServer.ts +++ b/packages/cli-v3/src/ipc/grpcServer.ts @@ -140,8 +140,13 @@ export class GrpcWorkerServer extends EventEmitter { ...(logMessage.exception && { exception: logMessage.exception }), }; + // Normalize log level to number (protobuf sends enum as string name) + const level = typeof logMessage.level === 'string' + ? LogLevel[logMessage.level as keyof typeof LogLevel] + : logMessage.level; + // Route to appropriate log level - switch (logMessage.level) { + switch (level) { case LogLevel.DEBUG: logger.debug('Python worker', logData); break; @@ -158,7 +163,7 @@ export class GrpcWorkerServer extends EventEmitter { } break; default: - logger.info('Python worker', logData); + logger.warn('Python worker (unknown log level)', { ...logData, receivedLevel: logMessage.level }); } } diff --git a/packages/cli-v3/src/python/pythonProcess.ts b/packages/cli-v3/src/python/pythonProcess.ts index a218896083..76294296f1 100644 --- a/packages/cli-v3/src/python/pythonProcess.ts +++ b/packages/cli-v3/src/python/pythonProcess.ts @@ -54,6 +54,7 @@ export class PythonProcess { ...process.env, ...this.options.env, PYTHONUNBUFFERED: "1", + PYTHONDONTWRITEBYTECODE: "1", // Disable .pyc files in dev to avoid stale cache TRIGGER_GRPC_ADDRESS: grpcAddress, }, stdio: ["pipe", "pipe", "pipe"], diff --git a/packages/cli-v3/src/python/pythonTaskRunner.ts b/packages/cli-v3/src/python/pythonTaskRunner.ts index dbf627332c..28b5b480e3 100644 --- a/packages/cli-v3/src/python/pythonTaskRunner.ts +++ b/packages/cli-v3/src/python/pythonTaskRunner.ts @@ -113,9 +113,17 @@ export class PythonTaskRunner { // Wait for completion or failure const result = await new Promise((resolve, reject) => { + // execution.run.maxDuration is in SECONDS from the platform, convert to milliseconds + const maxDurationMs = execution.run.maxDuration + ? execution.run.maxDuration * 1000 + : 300000; + logger.debug(`Setting task timeout to ${maxDurationMs}ms (${maxDurationMs/1000}s)`, { + maxDuration: execution.run.maxDuration, + runId: execution.run.id, + }); const timeout = setTimeout(() => { reject(new Error("Task execution timeout")); - }, execution.run.maxDuration ?? 300000); + }, maxDurationMs); grpcServer.on("TASK_RUN_COMPLETED", (message: any) => { clearTimeout(timeout); diff --git a/packages/python-sdk/trigger_sdk/task.py b/packages/python-sdk/trigger_sdk/task.py index dac8f096ff..7b6822ab8c 100644 --- a/packages/python-sdk/trigger_sdk/task.py +++ b/packages/python-sdk/trigger_sdk/task.py @@ -88,11 +88,14 @@ async def my_task(payload): retry_config = RetryConfig(**retry) if isinstance(retry, dict) else retry queue_config = QueueConfig(**queue) if isinstance(queue, dict) else queue + # Convert max_duration from seconds to milliseconds (TypeScript expects ms) + max_duration_ms = max_duration * 1000 if max_duration is not None else None + config = TaskConfig( id=id, retry=retry_config, queue=queue_config, - maxDuration=max_duration, + maxDuration=max_duration_ms, ) def decorator(fn: Callable[..., Any]) -> Task: diff --git a/packages/python-sdk/trigger_sdk/types.py b/packages/python-sdk/trigger_sdk/types.py index 75fad3c05b..732d5af512 100644 --- a/packages/python-sdk/trigger_sdk/types.py +++ b/packages/python-sdk/trigger_sdk/types.py @@ -24,7 +24,7 @@ class TaskConfig(BaseModel): id: str retry: Optional[RetryConfig] = None queue: Optional[QueueConfig] = None - maxDuration: Optional[int] = None # seconds + maxDuration: Optional[int] = None # milliseconds (converted from seconds in decorator) class TaskMetadata(BaseModel):