Merge pull request #1042 from UiPath/mj/fix-progress-reporting

fix: populate inputSchema and outputSchema for coded agents

Author: mjnovice

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.2.37"
+version = "2.2.38"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

src/uipath/_cli/_evals/_progress_reporter.py

@@ -537,12 +537,57 @@ def _serialize_justification(
 
         return justification
 
-    def _extract_agent_snapshot(self, entrypoint: str) -> StudioWebAgentSnapshot:
+    def _extract_agent_snapshot(self, entrypoint: str | None) -> StudioWebAgentSnapshot:
+        """Extract agent snapshot from entry points configuration or low-code agent file.
+
+        For coded agents, reads from entry-points.json configuration file.
+        For low-code agents (*.json files like agent.json), reads inputSchema
+        and outputSchema directly from the agent file.
+
+        Args:
+            entrypoint: The entrypoint file path to look up
+
+        Returns:
+            StudioWebAgentSnapshot with input and output schemas
+        """
+        if not entrypoint:
+            logger.warning(
+                "Entrypoint not provided - falling back to empty inputSchema "
+                "and outputSchema"
+            )
+            return StudioWebAgentSnapshot(input_schema={}, output_schema={})
+
         try:
+            # Check if entrypoint is a low-code agent JSON file (e.g., agent.json)
+            if entrypoint.endswith(".json"):
+                agent_file_path = os.path.join(os.getcwd(), entrypoint)
+                if os.path.exists(agent_file_path):
+                    with open(agent_file_path, "r") as f:
+                        agent_data = json.load(f)
+
+                    # Low-code agent files have inputSchema and outputSchema at root
+                    input_schema = agent_data.get("inputSchema", {})
+                    output_schema = agent_data.get("outputSchema", {})
+
+                    logger.debug(
+                        f"Extracted agent snapshot from low-code agent '{entrypoint}': "
+                        f"inputSchema={json.dumps(input_schema)}, "
+                        f"outputSchema={json.dumps(output_schema)}"
+                    )
+
+                    return StudioWebAgentSnapshot(
+                        input_schema=input_schema, output_schema=output_schema
+                    )
+
+            # Fall back to entry-points.json for coded agents
             entry_points_file_path = os.path.join(
                 os.getcwd(), str(UiPathConfig.entry_points_file_path)
             )
             if not os.path.exists(entry_points_file_path):
+                logger.debug(
+                    f"Entry points file not found at {entry_points_file_path}, "
+                    "using empty schemas"
+                )
                 return StudioWebAgentSnapshot(input_schema={}, output_schema={})
 
             with open(entry_points_file_path, "r") as f:
@@ -563,6 +608,12 @@ def _extract_agent_snapshot(self, entrypoint: str) -> StudioWebAgentSnapshot:
             input_schema = ep.get("input", {})
             output_schema = ep.get("output", {})
 
+            logger.debug(
+                f"Extracted agent snapshot for entrypoint '{entrypoint}': "
+                f"inputSchema={json.dumps(input_schema)}, "
+                f"outputSchema={json.dumps(output_schema)}"
+            )
+
             return StudioWebAgentSnapshot(
                 input_schema=input_schema, output_schema=output_schema
             )
@@ -724,6 +775,14 @@ def _update_eval_run_spec(
         # Both coded and legacy send payload directly at root level
         payload = inner_payload
 
+        # Log the payload for debugging eval run updates
+        agent_type = "coded" if is_coded else "low-code"
+        logger.debug(
+            f"Updating eval run (type={agent_type}): "
+            f"evalRunId={eval_run_id}, success={success}"
+        )
+        logger.debug(f"Full eval run update payload: {json.dumps(payload, indent=2)}")
+
         return RequestSpec(
             method="PUT",
             endpoint=Endpoint(
@@ -762,6 +821,16 @@ def _update_coded_eval_run_spec(
             "evaluatorRuns": evaluator_runs,
         }
 
+        # Log the payload for debugging coded eval run updates
+        agent_type = "coded" if is_coded else "low-code"
+        logger.debug(
+            f"Updating coded eval run (type={agent_type}): "
+            f"evalRunId={eval_run_id}, success={success}"
+        )
+        logger.debug(
+            f"Full coded eval run update payload: {json.dumps(payload, indent=2)}"
+        )
+
         return RequestSpec(
             method="PUT",
             endpoint=Endpoint(
@@ -826,6 +895,14 @@ def _create_eval_run_spec(
         # Both coded and legacy send payload directly at root level
         payload = inner_payload
 
+        # Log the payload for debugging eval run reporting
+        agent_type = "coded" if is_coded else "low-code"
+        logger.debug(
+            f"Creating eval run (type={agent_type}): "
+            f"evalSetRunId={eval_set_run_id}, evalItemId={eval_item.id}"
+        )
+        logger.debug(f"Full eval run payload: {json.dumps(payload, indent=2)}")
+
         return RequestSpec(
             method="POST",
             endpoint=Endpoint(
@@ -872,6 +949,16 @@ def _create_eval_set_run_spec(
         # Both coded and legacy send payload directly at root level
         payload = inner_payload
 
+        # Log the payload for debugging eval set run reporting
+        agent_type = "coded" if is_coded else "low-code"
+        logger.info(
+            f"Creating eval set run (type={agent_type}): "
+            f"evalSetId={eval_set_id}, "
+            f"inputSchema={json.dumps(payload.get('agentSnapshot', {}).get('inputSchema', {}))}, "
+            f"outputSchema={json.dumps(payload.get('agentSnapshot', {}).get('outputSchema', {}))}"
+        )
+        logger.debug(f"Full eval set run payload: {json.dumps(payload, indent=2)}")
+
         return RequestSpec(
             method="POST",
             endpoint=Endpoint(
@@ -926,6 +1013,17 @@ def _update_eval_set_run_spec(
         # Both coded and legacy send payload directly at root level
         payload = inner_payload
 
+        # Log the payload for debugging eval set run updates
+        agent_type = "coded" if is_coded else "low-code"
+        logger.info(
+            f"Updating eval set run (type={agent_type}): "
+            f"evalSetRunId={eval_set_run_id}, success={success}, "
+            f"evaluatorScores={json.dumps(payload.get('evaluatorScores', []))}"
+        )
+        logger.debug(
+            f"Full eval set run update payload: {json.dumps(payload, indent=2)}"
+        )
+
         return RequestSpec(
             method="PUT",
             endpoint=Endpoint(

tests/cli/eval/test_progress_reporter.py

@@ -551,3 +551,171 @@ def test_update_eval_set_run_spec_with_failure_legacy(self, progress_reporter):
         assert spec.json["evalSetRunId"] == "test-run-id"
         # Backend expects integer status
         assert spec.json["status"] == 3  # FAILED
+
+
+# Tests for agent snapshot extraction
+class TestAgentSnapshotExtraction:
+    """Tests for extracting agent snapshot with proper schema handling."""
+
+    def test_extract_agent_snapshot_reads_from_entry_points(
+        self, progress_reporter, tmp_path, monkeypatch
+    ):
+        """Test that agent snapshot reads schemas from entry points file."""
+        import os
+
+        # Create a temporary entry points file with full schemas
+        entry_points_data = {
+            "entryPoints": [
+                {
+                    "filePath": "test_agent",
+                    "uniqueId": "test-uuid",
+                    "type": "agent",
+                    "input": {
+                        "type": "object",
+                        "properties": {"query": {"type": "string"}},
+                    },
+                    "output": {
+                        "type": "object",
+                        "properties": {"response": {"type": "string"}},
+                    },
+                }
+            ]
+        }
+
+        entry_points_file = tmp_path / "entry-points.json"
+        with open(entry_points_file, "w") as f:
+            json.dump(entry_points_data, f)
+
+        # Change to the temp directory so the reporter finds the file
+        original_cwd = os.getcwd()
+        os.chdir(tmp_path)
+
+        try:
+            snapshot = progress_reporter._extract_agent_snapshot(
+                entrypoint="test_agent"
+            )
+
+            # Should read full schemas from entry points
+            assert snapshot.input_schema == {
+                "type": "object",
+                "properties": {"query": {"type": "string"}},
+            }
+            assert snapshot.output_schema == {
+                "type": "object",
+                "properties": {"response": {"type": "string"}},
+            }
+        finally:
+            os.chdir(original_cwd)
+
+    def test_extract_agent_snapshot_returns_empty_when_no_file(self, progress_reporter):
+        """Test that empty schemas are returned when entry points file doesn't exist."""
+        snapshot = progress_reporter._extract_agent_snapshot(
+            entrypoint="nonexistent_agent"
+        )
+
+        assert snapshot.input_schema == {}
+        assert snapshot.output_schema == {}
+
+    def test_extract_agent_snapshot_warns_when_entrypoint_is_none(
+        self, progress_reporter, caplog
+    ):
+        """Test that a warning is logged when entrypoint is None."""
+        import logging
+
+        with caplog.at_level(logging.WARNING):
+            snapshot = progress_reporter._extract_agent_snapshot(entrypoint=None)
+
+        assert snapshot.input_schema == {}
+        assert snapshot.output_schema == {}
+        assert "Entrypoint not provided" in caplog.text
+        assert "falling back to empty inputSchema" in caplog.text
+
+    def test_extract_agent_snapshot_warns_when_entrypoint_is_empty(
+        self, progress_reporter, caplog
+    ):
+        """Test that a warning is logged when entrypoint is empty string."""
+        import logging
+
+        with caplog.at_level(logging.WARNING):
+            snapshot = progress_reporter._extract_agent_snapshot(entrypoint="")
+
+        assert snapshot.input_schema == {}
+        assert snapshot.output_schema == {}
+        assert "Entrypoint not provided" in caplog.text
+
+    def test_extract_agent_snapshot_returns_empty_when_entrypoint_not_found(
+        self, progress_reporter, tmp_path
+    ):
+        """Test that empty schemas are returned when entrypoint is not in file."""
+        import os
+
+        # Create entry points file without the requested entrypoint
+        entry_points_data = {
+            "entryPoints": [
+                {
+                    "filePath": "other_agent",
+                    "uniqueId": "test-uuid",
+                    "type": "agent",
+                    "input": {"type": "object"},
+                    "output": {"type": "object"},
+                }
+            ]
+        }
+
+        entry_points_file = tmp_path / "entry-points.json"
+        with open(entry_points_file, "w") as f:
+            json.dump(entry_points_data, f)
+
+        original_cwd = os.getcwd()
+        os.chdir(tmp_path)
+
+        try:
+            snapshot = progress_reporter._extract_agent_snapshot(
+                entrypoint="nonexistent_agent"
+            )
+
+            assert snapshot.input_schema == {}
+            assert snapshot.output_schema == {}
+        finally:
+            os.chdir(original_cwd)
+
+    def test_agent_snapshot_serializes_with_camel_case(
+        self, progress_reporter, tmp_path
+    ):
+        """Test that agent snapshot serializes to correct JSON format with camelCase."""
+        import os
+
+        entry_points_data = {
+            "entryPoints": [
+                {
+                    "filePath": "test_agent",
+                    "uniqueId": "test-uuid",
+                    "type": "agent",
+                    "input": {"type": "object", "properties": {}},
+                    "output": {"type": "object", "properties": {}},
+                }
+            ]
+        }
+
+        entry_points_file = tmp_path / "entry-points.json"
+        with open(entry_points_file, "w") as f:
+            json.dump(entry_points_data, f)
+
+        original_cwd = os.getcwd()
+        os.chdir(tmp_path)
+
+        try:
+            snapshot = progress_reporter._extract_agent_snapshot(
+                entrypoint="test_agent"
+            )
+
+            # Serialize using pydantic
+            serialized = snapshot.model_dump(by_alias=True)
+
+            # Should have camelCase keys
+            assert "inputSchema" in serialized
+            assert "outputSchema" in serialized
+            assert serialized["inputSchema"] == {"type": "object", "properties": {}}
+            assert serialized["outputSchema"] == {"type": "object", "properties": {}}
+        finally:
+            os.chdir(original_cwd)