feat: compare version model evaluations in the CLI

CHANGELOG.md

@@ -2,6 +2,25 @@
 
 All notable changes to this project will be documented in this file.
 
+## 1.5.1
+
+### Added
+
+- Model evaluation comparison for a dataset version
+  ([#529](https://github.com/roboflow/roboflow-python/pull/529)):
+  - `Workspace.compare_model_evaluations(project, version, frontier_metric=None)`
+    — compare test-set accuracy and median latency, including Pareto frontier
+    membership and reasons models are excluded.
+  - `roboflow --workspace <workspace> eval compare --project <project> --version <N>`
+    — display the comparison as a table; use `--json` for the public API response.
+    Use `--frontier-metric` to choose the metric for frontier membership.
+  - Both surfaces read existing results without starting evaluations.
+
+### Changed
+
+- All `roboflow eval` commands now return exit code `2` for HTTP 401/403
+  access errors (previously `1`).
+
 ## 1.5.0
 
 ### Added

CLI-COMMANDS.md

@@ -125,6 +125,7 @@ roboflow --json train results my-project/3 | jq -r .modelGroup
 roboflow model list -p my-project --group rfdetrNasGroup-3
 
 # Star a NAS-trained model (triggers TRT compile for its recommended hardware):
+#   Also starts model evaluation when the workspace has Model Evaluation access.
 #   --json train results … gives you the modelId per row.
 roboflow model star <modelId>
 roboflow model star <modelId> --unstar
@@ -385,11 +386,25 @@ roboflow eval recommendations <eval-id> --json
 ```
 
 Exit codes are stable per error class so scripts and agents can react
-without parsing message strings: `3` for `model_eval_not_found` (404),
+without parsing message strings: `2` for authentication or access errors (401/403),
+`3` for `model_eval_not_found` (404),
 `4` for `model_eval_not_done` (409 — eval still running), `5` for
 `invalid_split` / `invalid_confidence` (400). Requires the
 `model-eval:read` scope on the api key.
 
+### Compare Models
+
+```bash
+# Compare accuracy and latency for one version.
+roboflow --workspace my-workspace eval compare --project my-project --version 3
+
+# Choose the Pareto frontier metric; return all metrics and exclusion reasons.
+roboflow eval compare --project my-project --version 3 --frontier-metric mAP5095 --json
+```
+
+Reads existing evaluations; does not start new ones. Requires `model-eval:read`
+and workspace Model Evaluation access.
+
 ### Workspace stats and billing
 
 ```bash

roboflow/__init__.py

@@ -21,7 +21,7 @@
     CLIPModel = None  # type: ignore[assignment,misc]
     GazeModel = None  # type: ignore[assignment,misc]
 
-__version__ = "1.5.0"
+__version__ = "1.5.1"
 
 
 def check_key(api_key, model, notebook, num_retries=0):

roboflow/adapters/rfapi.py

@@ -2110,6 +2110,10 @@ class ModelEvalNotDoneError(RoboflowError):
     """Raised when reading panel data for an eval whose status is not ``done`` (HTTP 409)."""
 
 
+class ModelEvalAccessError(RoboflowError):
+    """Raised when an evaluation read is not authorized (HTTP 401 or 403)."""
+
+
 class InvalidSplitError(RoboflowError):
     """Raised when ``split`` is not one of the accepted values (HTTP 400)."""
 
@@ -2149,6 +2153,8 @@ def _model_eval_error_for(response):
         "invalid_confidence": InvalidConfidenceError,
     }
     cls = cls_by_code.get(code or "")
+    if response.status_code in (401, 403):
+        return ModelEvalAccessError(message)
     if cls is not None:
         return cls(message)
     if response.status_code == 404:
@@ -2172,6 +2178,23 @@ def _eval_get(api_key, workspace_url, path, params=None):
     return response.json()
 
 
+def compare_model_evals(
+    api_key: str,
+    workspace_url: str,
+    *,
+    project: str,
+    version: Union[str, int],
+    frontier_metric: Optional[str] = None,
+) -> dict:
+    """GET /{workspace}/model-evals/compare — compare models on a dataset version."""
+    return _eval_get(
+        api_key,
+        workspace_url,
+        "/compare",
+        params={"project": project, "version": version, "frontierMetric": frontier_metric},
+    )
+
+
 def list_model_evals(
     api_key: str,
     workspace_url: str,

roboflow/cli/handlers/eval.py

@@ -24,6 +24,24 @@
 # ---------------------------------------------------------------------------
 
 
+@eval_app.command("compare")
+def compare_evals_cmd(
+    ctx: typer.Context,
+    project: Annotated[str, typer.Option("-p", "--project", help="Project slug")],
+    version: Annotated[int, typer.Option("-v", "--version", min=1, help="Dataset version number")],
+    frontier_metric: Annotated[
+        Optional[str],
+        typer.Option(
+            "--frontier-metric",
+            help="Frontier metric: mAP, mAP5095, mAP75, mIoU, precision, recall, f1 (default: project default)",
+        ),
+    ] = None,
+) -> None:
+    """Compare test-set accuracy and median latency without starting evaluations."""
+    args = ctx_to_args(ctx, project=project, version=version, frontier_metric=frontier_metric)
+    _compare_evals(args)
+
+
 @eval_app.command("list")
 def list_evals_cmd(
     ctx: typer.Context,
@@ -171,6 +189,8 @@ def _eval_error_exit_code(exc: Exception) -> int:
     """
     from roboflow.adapters import rfapi
 
+    if isinstance(exc, rfapi.ModelEvalAccessError):
+        return 2
     if isinstance(exc, rfapi.ModelEvalNotFoundError):
         return 3
     if isinstance(exc, rfapi.ModelEvalNotDoneError):
@@ -184,6 +204,8 @@ def _hint_for(exc: Exception) -> Optional[str]:
     """Per-error actionable hint shown alongside the message in non-JSON mode."""
     from roboflow.adapters import rfapi
 
+    if isinstance(exc, rfapi.ModelEvalAccessError):
+        return "Check your API key, its model-eval:read scope, and workspace Model Evaluation access."
     if isinstance(exc, rfapi.ModelEvalNotFoundError):
         return "Run 'roboflow eval list' to see eval ids in this workspace."
     if isinstance(exc, rfapi.ModelEvalNotDoneError):
@@ -195,6 +217,60 @@ def _hint_for(exc: Exception) -> Optional[str]:
     return None
 
 
+def _compare_evals(args):  # noqa: ANN001
+    from roboflow.adapters import rfapi
+    from roboflow.cli._output import output, output_error
+    from roboflow.cli._table import format_table
+
+    resolved = _resolve(args)
+    if not resolved:
+        return
+    workspace_url, api_key = resolved
+    try:
+        comparison = rfapi.compare_model_evals(
+            api_key,
+            workspace_url,
+            project=args.project,
+            version=args.version,
+            frontier_metric=args.frontier_metric,
+        )
+    except Exception as exc:
+        output_error(
+            args,
+            str(exc),
+            hint=(
+                _hint_for(exc)
+                if isinstance(exc, rfapi.ModelEvalAccessError)
+                else "Check the project, version, frontier metric, and workspace access."
+            ),
+            exit_code=_eval_error_exit_code(exc),
+        )
+        return
+    if args.json:
+        output(args, comparison)
+        return
+    frontier_metric = comparison.get("frontierMetric")
+    rows = []
+    for model in comparison.get("models", []):
+        accuracy = model.get("metrics", {}).get(frontier_metric) if frontier_metric else None
+        latency = model.get("medianLatencyMs")
+        rows.append(
+            {
+                "model": model["modelId"],
+                "accuracy": f"{accuracy * 100:.1f}%" if accuracy is not None else "",
+                "latency": f"{latency:.2f}" if latency is not None else "",
+                "frontier": "Yes" if model.get("onFrontier") else "",
+                "exclusion": model.get("exclusionReason") or "",
+            }
+        )
+    table = format_table(
+        rows,
+        columns=["model", "accuracy", "latency", "frontier", "exclusion"],
+        headers=["MODEL", frontier_metric or "ACCURACY", "MEDIAN LATENCY (ms)", "FRONTIER", "EXCLUSION"],
+    )
+    output(args, comparison, text=table)
+
+
 def _list_evals(args):  # noqa: ANN001
     from roboflow.adapters import rfapi
     from roboflow.cli._output import output, output_error

roboflow/core/workspace.py

@@ -8,7 +8,7 @@
 import tempfile
 import time
 import zipfile
-from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union
 
 import requests
 from requests.exceptions import HTTPError
@@ -1572,6 +1572,32 @@ def upload_vision_event_image(
     # Model evaluations
     # -----------------------------------------------------------------
 
+    def compare_model_evaluations(
+        self,
+        project: str,
+        version: Union[str, int],
+        *,
+        frontier_metric: Optional[str] = None,
+    ) -> dict:
+        """Compare model accuracy and median latency for a dataset version.
+
+        Args:
+            project: Project URL slug.
+            version: Dataset version number.
+            frontier_metric: Metric for frontier membership. The server uses
+                the project default when this value is not specified.
+
+        Returns:
+            The public model comparison response.
+        """
+        return rfapi.compare_model_evals(
+            self.__api_key,
+            self.url,
+            project=project,
+            version=version,
+            frontier_metric=frontier_metric,
+        )
+
     def evals(
         self,
         *,

tests/adapters/test_rfapi_model_evals.py

@@ -220,5 +220,31 @@ def test_non_json_body_falls_back_to_text(self, mock_get):
         self.assertIn("Bad Gateway", str(ctx.exception))
 
 
+class TestCompareModelEvals(unittest.TestCase):
+    @patch("roboflow.adapters.rfapi.requests.get")
+    def test_compare_returns_public_result_and_sends_frontier_metric(self, mock_get):
+        comparison = {
+            "project": "chess",
+            "version": "131",
+            "frontierMetric": "mAP5095",
+            "availableMetrics": ["mAP", "mAP5095"],
+            "models": [],
+        }
+        mock_get.return_value = _resp(200, comparison)
+
+        result = rfapi.compare_model_evals("k", "ws", project="chess", version="131", frontier_metric="mAP5095")
+
+        self.assertEqual(result, comparison)
+        mock_get.assert_called_once_with(
+            f"{API_URL}/ws/model-evals/compare",
+            params={
+                "api_key": "k",
+                "project": "chess",
+                "version": "131",
+                "frontierMetric": "mAP5095",
+            },
+        )
+
+
 if __name__ == "__main__":
     unittest.main()