changed scenario and scenario smoketests to update existing scenarios and add metadata tag instead of creating new ones every time (workaround for lack of cleanup option)

sid-rl · sid-rl · commit dc8a68ede01f · 2025-12-04T12:06:55.000-08:00
diff --git a/tests/smoketests/sdk/test_async_scenario.py b/tests/smoketests/sdk/test_async_scenario.py
@@ -2,16 +2,30 @@
 
 from __future__ import annotations
 
+import datetime
+from typing import Optional
+
 import pytest
 
 from runloop_api_client.sdk import AsyncRunloopSDK
+from runloop_api_client.sdk.async_scenario import AsyncScenario
 
 pytestmark = [pytest.mark.smoketest]
 
 TWO_MINUTE_TIMEOUT = 120
 FIVE_MINUTE_TIMEOUT = 300
 
 
+async def find_scenario_by_name_async(sdk_client: AsyncRunloopSDK, name: str) -> Optional[AsyncScenario]:
+    """Find an existing scenario by name, returns None if not found."""
+    scenarios = await sdk_client.scenario.list(limit=100)
+    for scenario in scenarios:
+        info = await scenario.get_info()
+        if info.name == name:
+            return scenario
+    return None
+
+
 class TestAsyncScenarioRetrieval:
     """Test async scenario retrieval operations."""
 
@@ -124,3 +138,136 @@ async def test_scenario_run_and_await_env_ready(self, async_sdk_client: AsyncRun
                 await run.cancel()
             except Exception:
                 pass
+
+
+class TestAsyncScenarioBuilder:
+    """Test async scenario builder operations.
+
+    These tests use fixed scenario names and reuse existing scenarios when possible
+    to avoid accumulating test scenarios (since scenario deletion is not yet available).
+    """
+
+    # Metadata for identifying test scenarios that can be cleaned up later
+    CLEANUP_METADATA = {"smoketest": "true", "sdk_smoketest": "true", "cleanup_safe": "true"}
+
+    # Fixed scenario names for reuse across test runs
+    BASIC_SCENARIO_NAME = "sdk-smoketest-async-builder-basic"
+    ENV_SCENARIO_NAME = "sdk-smoketest-async-builder-env"
+    FLUENT_SCENARIO_NAME = "sdk-smoketest-async-builder-fluent"
+
+    @pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
+    async def test_builder_create_scenario(self, async_sdk_client: AsyncRunloopSDK) -> None:
+        """Test creating a scenario with the async builder.
+
+        Uses a fixed name and reuses existing scenario if found.
+        """
+        # Check if scenario already exists
+        existing = await find_scenario_by_name_async(async_sdk_client, self.BASIC_SCENARIO_NAME)
+        if existing:
+            # Scenario exists, update metadata to exercise update path
+            updated_metadata = {
+                **self.CLEANUP_METADATA,
+                "last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            }
+            await existing.update(metadata=updated_metadata)
+            info = await existing.get_info()
+            assert info.name == self.BASIC_SCENARIO_NAME
+            assert info.input_context is not None
+            return
+
+        # Create new scenario
+        builder = async_sdk_client.scenario.builder(self.BASIC_SCENARIO_NAME)
+        builder.with_problem_statement("This is a test scenario created by the async SDK builder smoke test.")
+        builder.add_bash_scorer(
+            "test-scorer",
+            bash_script="echo 'score=1.0'",
+            weight=1.0,
+        )
+        builder.with_metadata(self.CLEANUP_METADATA)
+
+        scenario = await builder.push()
+
+        assert scenario.id is not None
+        info = await scenario.get_info()
+        assert info.name == self.BASIC_SCENARIO_NAME
+        assert info.input_context is not None
+
+    @pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
+    async def test_builder_with_environment(self, async_sdk_client: AsyncRunloopSDK) -> None:
+        """Test creating a scenario with environment configuration.
+
+        Uses a fixed name and reuses existing scenario if found.
+        """
+        # Check if scenario already exists
+        existing = await find_scenario_by_name_async(async_sdk_client, self.ENV_SCENARIO_NAME)
+        if existing:
+            # Update metadata to exercise update path
+            updated_metadata = {
+                **self.CLEANUP_METADATA,
+                "last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            }
+            await existing.update(metadata=updated_metadata)
+            info = await existing.get_info()
+            assert info.name == self.ENV_SCENARIO_NAME
+            return
+
+        # Get a blueprint to use
+        blueprints = await async_sdk_client.blueprint.list(limit=1)
+        if not blueprints:
+            pytest.skip("No blueprints available for environment test")
+
+        blueprint_id = blueprints[0].id
+
+        builder = (
+            async_sdk_client.scenario.builder(self.ENV_SCENARIO_NAME)
+            .from_blueprint_id(blueprint_id)
+            .with_working_directory("/workspace")
+            .with_problem_statement("Test scenario with environment configuration.")
+            .add_command_scorer("check", command="echo 'score=1.0'", weight=1.0)
+            .with_metadata(self.CLEANUP_METADATA)
+        )
+
+        scenario = await builder.push()
+
+        info = await scenario.get_info()
+        assert info.name == self.ENV_SCENARIO_NAME
+        if info.environment:
+            assert info.environment.blueprint_id == blueprint_id
+
+    @pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
+    async def test_builder_fluent_api(self, async_sdk_client: AsyncRunloopSDK) -> None:
+        """Test async builder fluent API chaining.
+
+        Uses a fixed name and reuses existing scenario if found.
+        """
+        # Check if scenario already exists
+        existing = await find_scenario_by_name_async(async_sdk_client, self.FLUENT_SCENARIO_NAME)
+        if existing:
+            # Update metadata to exercise update path
+            updated_metadata = {
+                **self.CLEANUP_METADATA,
+                "last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+                "test_type": "fluent_api",
+            }
+            await existing.update(metadata=updated_metadata)
+            info = await existing.get_info()
+            assert info.name == self.FLUENT_SCENARIO_NAME
+            if info.scoring_contract and info.scoring_contract.scoring_function_parameters:
+                assert len(info.scoring_contract.scoring_function_parameters) == 2
+            return
+
+        # Create new scenario with fluent API
+        scenario = await (
+            async_sdk_client.scenario.builder(self.FLUENT_SCENARIO_NAME)
+            .with_problem_statement("Async fluent API test scenario.")
+            .with_additional_context({"hint": "This is a hint"})
+            .add_bash_scorer("scorer1", bash_script="echo 'score=0.5'", weight=1.0)
+            .add_command_scorer("scorer2", command="echo 'score=1.0'", weight=2.0)
+            .with_metadata({**self.CLEANUP_METADATA, "test_type": "fluent_api"})
+            .push()
+        )
+
+        info = await scenario.get_info()
+        assert info.name == self.FLUENT_SCENARIO_NAME
+        if info.scoring_contract and info.scoring_contract.scoring_function_parameters:
+            assert len(info.scoring_contract.scoring_function_parameters) == 2
diff --git a/tests/smoketests/sdk/test_scenario.py b/tests/smoketests/sdk/test_scenario.py
@@ -2,16 +2,30 @@
 
 from __future__ import annotations
 
+import datetime
+from typing import Optional
+
 import pytest
 
 from runloop_api_client.sdk import RunloopSDK
+from runloop_api_client.sdk.scenario import Scenario
 
 pytestmark = [pytest.mark.smoketest]
 
 TWO_MINUTE_TIMEOUT = 120
 FIVE_MINUTE_TIMEOUT = 300
 
 
+def find_scenario_by_name(sdk_client: RunloopSDK, name: str) -> Optional[Scenario]:
+    """Find an existing scenario by name, returns None if not found."""
+    scenarios = sdk_client.scenario.list(limit=100)
+    for scenario in scenarios:
+        info = scenario.get_info()
+        if info.name == name:
+            return scenario
+    return None
+
+
 class TestScenarioRetrieval:
     """Test scenario retrieval operations."""
 
@@ -124,3 +138,136 @@ def test_scenario_run_and_await_env_ready(self, sdk_client: RunloopSDK) -> None:
                 run.cancel()
             except Exception:
                 pass
+
+
+class TestScenarioBuilder:
+    """Test scenario builder operations.
+
+    These tests use fixed scenario names and reuse existing scenarios when possible
+    to avoid accumulating test scenarios (since scenario deletion is not yet available).
+    """
+
+    # Metadata for identifying test scenarios that can be cleaned up later
+    CLEANUP_METADATA = {"smoketest": "true", "sdk_smoketest": "true", "cleanup_safe": "true"}
+
+    # Fixed scenario names for reuse across test runs
+    BASIC_SCENARIO_NAME = "sdk-smoketest-builder-basic"
+    ENV_SCENARIO_NAME = "sdk-smoketest-builder-env"
+    FLUENT_SCENARIO_NAME = "sdk-smoketest-builder-fluent"
+
+    @pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
+    def test_builder_create_scenario(self, sdk_client: RunloopSDK) -> None:
+        """Test creating a scenario with the builder.
+
+        Uses a fixed name and reuses existing scenario if found.
+        """
+        # Check if scenario already exists
+        existing = find_scenario_by_name(sdk_client, self.BASIC_SCENARIO_NAME)
+        if existing:
+            # Scenario exists, update metadata to exercise update path
+            updated_metadata = {
+                **self.CLEANUP_METADATA,
+                "last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            }
+            existing.update(metadata=updated_metadata)
+            info = existing.get_info()
+            assert info.name == self.BASIC_SCENARIO_NAME
+            assert info.input_context is not None
+            return
+
+        # Create new scenario
+        builder = sdk_client.scenario.builder(self.BASIC_SCENARIO_NAME)
+        builder.with_problem_statement("This is a test scenario created by the SDK builder smoke test.")
+        builder.add_bash_scorer(
+            "test-scorer",
+            bash_script="echo 'score=1.0'",
+            weight=1.0,
+        )
+        builder.with_metadata(self.CLEANUP_METADATA)
+
+        scenario = builder.push()
+
+        assert scenario.id is not None
+        info = scenario.get_info()
+        assert info.name == self.BASIC_SCENARIO_NAME
+        assert info.input_context is not None
+
+    @pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
+    def test_builder_with_environment(self, sdk_client: RunloopSDK) -> None:
+        """Test creating a scenario with environment configuration.
+
+        Uses a fixed name and reuses existing scenario if found.
+        """
+        # Check if scenario already exists
+        existing = find_scenario_by_name(sdk_client, self.ENV_SCENARIO_NAME)
+        if existing:
+            # Update metadata to exercise update path
+            updated_metadata = {
+                **self.CLEANUP_METADATA,
+                "last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            }
+            existing.update(metadata=updated_metadata)
+            info = existing.get_info()
+            assert info.name == self.ENV_SCENARIO_NAME
+            return
+
+        # Get a blueprint to use
+        blueprints = sdk_client.blueprint.list(limit=1)
+        if not blueprints:
+            pytest.skip("No blueprints available for environment test")
+
+        blueprint_id = blueprints[0].id
+
+        builder = (
+            sdk_client.scenario.builder(self.ENV_SCENARIO_NAME)
+            .from_blueprint_id(blueprint_id)
+            .with_working_directory("/workspace")
+            .with_problem_statement("Test scenario with environment configuration.")
+            .add_command_scorer("check", command="echo 'score=1.0'", weight=1.0)
+            .with_metadata(self.CLEANUP_METADATA)
+        )
+
+        scenario = builder.push()
+
+        info = scenario.get_info()
+        assert info.name == self.ENV_SCENARIO_NAME
+        if info.environment:
+            assert info.environment.blueprint_id == blueprint_id
+
+    @pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
+    def test_builder_fluent_api(self, sdk_client: RunloopSDK) -> None:
+        """Test builder fluent API chaining.
+
+        Uses a fixed name and reuses existing scenario if found.
+        """
+        # Check if scenario already exists
+        existing = find_scenario_by_name(sdk_client, self.FLUENT_SCENARIO_NAME)
+        if existing:
+            # Update metadata to exercise update path
+            updated_metadata = {
+                **self.CLEANUP_METADATA,
+                "last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+                "test_type": "fluent_api",
+            }
+            existing.update(metadata=updated_metadata)
+            info = existing.get_info()
+            assert info.name == self.FLUENT_SCENARIO_NAME
+            if info.scoring_contract and info.scoring_contract.scoring_function_parameters:
+                assert len(info.scoring_contract.scoring_function_parameters) == 2
+            return
+
+        # Create new scenario with fluent API
+        scenario = (
+            sdk_client.scenario.builder(self.FLUENT_SCENARIO_NAME)
+            .with_problem_statement("Fluent API test scenario.")
+            .with_additional_context({"hint": "This is a hint"})
+            .add_bash_scorer("scorer1", bash_script="echo 'score=0.5'", weight=1.0)
+            .add_command_scorer("scorer2", command="echo 'score=1.0'", weight=2.0)
+            .with_metadata({**self.CLEANUP_METADATA, "test_type": "fluent_api"})
+            .push()
+        )
+
+        info = scenario.get_info()
+        assert info.name == self.FLUENT_SCENARIO_NAME
+        if info.scoring_contract and info.scoring_contract.scoring_function_parameters:
+            assert len(info.scoring_contract.scoring_function_parameters) == 2
