From 226becee76429e6b816db9829f284fad67be4aa6 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Wed, 16 Sep 2026 15:01:06 -0230 Subject: [PATCH 1/9] feat: compare version model evaluations in the CLI --- CLI-COMMANDS.md | 5 + roboflow/adapters/rfapi.py | 16 ++ roboflow/cli/handlers/eval.py | 59 +++++++ tests/adapters/test_rfapi_model_evals.py | 23 +++ tests/cli/test_eval_handler.py | 105 ++++++++++++ tests/fixtures/model_eval_comparison.json | 194 ++++++++++++++++++++++ 6 files changed, 402 insertions(+) create mode 100644 tests/fixtures/model_eval_comparison.json diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 10ec349a..c0f2f286 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -367,6 +367,11 @@ view. Items left in Trash are cleaned up automatically after 30 days. ### Inspect model evaluations +Compare models on a Dataset Version with `roboflow eval compare --project chess-pieces --version 131`. +Use `--json` for the full result, including server-computed frontier membership, exclusions, and current evaluation attempts. +The command is read-only. It needs `model-eval:read` access and Model Comparison enabled for the Workspace. +See the [Model Comparison reference](https://docs.roboflow.com/train/model-comparison) for the response contract and all command options. + ```bash # List evals in the workspace; filter by project, version, model, or status. roboflow eval list --status done --limit 10 diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index e29b2b2e..ec0adad2 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -2110,6 +2110,10 @@ class ModelEvalNotDoneError(RoboflowError): """Raised when reading panel data for an eval whose status is not ``done`` (HTTP 409).""" +class ModelEvalAccessError(RoboflowError): + """Raised when an evaluation read is not authorized (HTTP 401 or 403).""" + + class InvalidSplitError(RoboflowError): """Raised when ``split`` is not one of the accepted values (HTTP 400).""" @@ -2149,6 +2153,8 @@ def _model_eval_error_for(response): "invalid_confidence": InvalidConfidenceError, } cls = cls_by_code.get(code or "") + if response.status_code in (401, 403): + return ModelEvalAccessError(message) if cls is not None: return cls(message) if response.status_code == 404: @@ -2172,6 +2178,16 @@ def _eval_get(api_key, workspace_url, path, params=None): return response.json() +def compare_model_evals( + api_key: str, + workspace_url: str, + *, + project: str, + version: Union[str, int], +) -> dict: + return _eval_get(api_key, workspace_url, "/compare", params={"project": project, "version": version}) + + def list_model_evals( api_key: str, workspace_url: str, diff --git a/roboflow/cli/handlers/eval.py b/roboflow/cli/handlers/eval.py index 0e41cc90..ce234c57 100644 --- a/roboflow/cli/handlers/eval.py +++ b/roboflow/cli/handlers/eval.py @@ -24,6 +24,63 @@ # --------------------------------------------------------------------------- +@eval_app.command("compare") +def compare_evals_cmd( + ctx: typer.Context, + project: Annotated[str, typer.Option("-p", "--project", help="Project slug")], + version: Annotated[int, typer.Option("-v", "--version", min=1, help="Dataset version number")], +) -> None: + """Compare test-set accuracy and median latency without starting evaluations.""" + from roboflow.adapters import rfapi + from roboflow.cli._output import output, output_error + from roboflow.cli._table import format_table + + args = ctx_to_args(ctx, project=project, version=version) + resolved = _resolve(args) + if not resolved: + return + workspace_url, api_key = resolved + try: + comparison = rfapi.compare_model_evals(api_key, workspace_url, project=project, version=version) + except Exception as exc: + output_error( + args, + str(exc), + hint=( + "Check the project, version, and workspace access. Model Comparison must be enabled for the workspace." + ), + exit_code=_eval_error_exit_code(exc), + ) + return + if args.json: + output(args, comparison) + return + metric = comparison.get("metric") or {} + names = {model["id"]: model["name"] for model in comparison.get("models", [])} + rows = [] + for candidate in comparison.get("candidates", []): + accuracy = candidate.get("accuracy") + latency = candidate.get("medianLatencyMs") + rows.append( + { + "model": names.get(candidate["modelId"], candidate["modelId"]), + "accuracy": f"{accuracy * 100:.1f}%" if accuracy is not None else "", + "latency": f"{latency:.2f}" if latency is not None else "", + "frontier": "Yes" if candidate.get("onFrontier") else "", + "exclusion": candidate.get("exclusionReason") or "", + } + ) + table = format_table( + rows, + columns=["model", "accuracy", "latency", "frontier", "exclusion"], + headers=["MODEL", metric.get("label", "ACCURACY"), "MEDIAN LATENCY (ms)", "FRONTIER", "EXCLUSION"], + ) + lines = [f"Test set | Serving device: {comparison.get('servingDevice', '')}", table] + if comparison.get("appUrl"): + lines.append(comparison["appUrl"]) + output(args, comparison, text="\n".join(lines)) + + @eval_app.command("list") def list_evals_cmd( ctx: typer.Context, @@ -171,6 +228,8 @@ def _eval_error_exit_code(exc: Exception) -> int: """ from roboflow.adapters import rfapi + if isinstance(exc, rfapi.ModelEvalAccessError): + return 2 if isinstance(exc, rfapi.ModelEvalNotFoundError): return 3 if isinstance(exc, rfapi.ModelEvalNotDoneError): diff --git a/tests/adapters/test_rfapi_model_evals.py b/tests/adapters/test_rfapi_model_evals.py index 41cc6942..e0dac4f0 100644 --- a/tests/adapters/test_rfapi_model_evals.py +++ b/tests/adapters/test_rfapi_model_evals.py @@ -220,5 +220,28 @@ def test_non_json_body_falls_back_to_text(self, mock_get): self.assertIn("Bad Gateway", str(ctx.exception)) +class TestCompareModelEvals(unittest.TestCase): + @patch("roboflow.adapters.rfapi.requests.get") + def test_compare_returns_complete_server_result(self, mock_get): + comparison = { + "project": "chess", + "version": {"id": "131", "projectId": "chess", "projectType": "object-detection"}, + "metric": {"key": "mAP", "label": "mAP@50", "unit": "ratio"}, + "candidates": [{"modelId": f"ws/chess-{index}", "onFrontier": False} for index in range(201)], + "evaluations": [], + "models": [], + "trainings": [], + } + mock_get.return_value = _resp(200, comparison) + + result = rfapi.compare_model_evals("k", "ws", project="chess", version="131") + + self.assertEqual(result, comparison) + mock_get.assert_called_once_with( + f"{API_URL}/ws/model-evals/compare", + params={"api_key": "k", "project": "chess", "version": "131"}, + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/cli/test_eval_handler.py b/tests/cli/test_eval_handler.py index 0d4a4a30..11a50c19 100644 --- a/tests/cli/test_eval_handler.py +++ b/tests/cli/test_eval_handler.py @@ -409,5 +409,110 @@ def test_exit_codes(self) -> None: self.assertEqual(_eval_error_exit_code(exc), expected) +class TestEvalCompareCommand(unittest.TestCase): + @patch("roboflow.adapters.rfapi.requests.get") + def test_json_preserves_the_full_comparison(self, mock_get): + from pathlib import Path + from unittest.mock import MagicMock + + comparison = json.loads((Path(__file__).parents[1] / "fixtures/model_eval_comparison.json").read_text()) + mock_get.return_value = MagicMock(status_code=200) + mock_get.return_value.json.return_value = comparison + + result = runner.invoke( + app, + [ + "--api-key", + "k", + "--workspace", + "ws", + "--json", + "eval", + "compare", + "--project", + "chess", + "--version", + "131", + ], + ) + + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(json.loads(result.stdout), comparison) + self.assertEqual(mock_get.call_count, 1) + + @patch("roboflow.adapters.rfapi.requests.get") + def test_text_shows_server_frontier_and_exclusion_with_zero_values(self, mock_get): + from unittest.mock import MagicMock + + mock_get.return_value = MagicMock(status_code=200) + mock_get.return_value.json.return_value = { + "metric": {"key": "mAP", "label": "mAP@50", "unit": "ratio"}, + "servingDevice": "T4", + "models": [ + {"id": "ws/chess-fast", "name": "Fast model"}, + {"id": "ws/chess-old", "name": "Old model"}, + ], + "candidates": [ + {"modelId": "ws/chess-fast", "eligible": True, "accuracy": 0, "medianLatencyMs": 0, "onFrontier": True}, + { + "modelId": "ws/chess-old", + "eligible": False, + "accuracy": 0.9, + "medianLatencyMs": None, + "onFrontier": False, + "exclusionReason": "latency_unavailable", + }, + ], + "appUrl": "https://app.roboflow.com/ws/chess/131", + } + result = runner.invoke( + app, + ["--api-key", "k", "--workspace", "ws", "eval", "compare", "--project", "chess", "--version", "131"], + ) + + self.assertEqual(result.exit_code, 0, result.output) + for text in [ + "MODEL", + "mAP@50", + "MEDIAN LATENCY (ms)", + "FRONTIER", + "EXCLUSION", + "Fast model", + "0.0%", + "0.00", + "Yes", + "latency_unavailable", + "https://app.roboflow.com/ws/chess/131", + ]: + self.assertIn(text, result.stdout) + + @patch("roboflow.adapters.rfapi.requests.get") + def test_permission_failure_is_a_structured_auth_error(self, mock_get): + from unittest.mock import MagicMock + + mock_get.return_value = MagicMock(status_code=403, text="Comparison access denied") + mock_get.return_value.json.return_value = {"error": "forbidden", "message": "Comparison access denied"} + result = runner.invoke( + app, + [ + "--api-key", + "k", + "--workspace", + "ws", + "--json", + "eval", + "compare", + "--project", + "chess", + "--version", + "131", + ], + ) + + self.assertEqual(result.exit_code, 2) + self.assertEqual(json.loads(result.stderr)["error"]["message"], "Comparison access denied") + self.assertEqual(result.stdout, "") + + if __name__ == "__main__": unittest.main() diff --git a/tests/fixtures/model_eval_comparison.json b/tests/fixtures/model_eval_comparison.json new file mode 100644 index 00000000..9fcbd307 --- /dev/null +++ b/tests/fixtures/model_eval_comparison.json @@ -0,0 +1,194 @@ +{ + "project": "chess", + "version": { + "id": "131", + "projectId": "chess", + "projectType": "object-detection" + }, + "metric": { + "key": "mAP", + "label": "mAP@50", + "unit": "ratio" + }, + "servingDevice": "T4", + "models": [ + { + "id": "acme/chess-fast", + "name": "Fast model", + "url": "acme/chess-fast", + "projectId": "chess", + "versionId": "131", + "trainingId": "training-fast", + "modelType": "rfdetr-small", + "origin": "roboflow-train", + "isNasChild": false + }, + { + "id": "acme/chess-dominated", + "name": "Dominated model", + "url": "acme/chess-dominated", + "projectId": "chess", + "versionId": "131", + "trainingId": "training-dominated", + "modelType": "rfdetr-small", + "origin": "roboflow-train", + "isNasChild": false + }, + { + "id": "acme/chess-missing-latency", + "name": "Previously evaluated model", + "url": "acme/chess-missing-latency", + "projectId": "chess", + "versionId": "131", + "trainingId": "training-missing-latency", + "modelType": "rfdetr-small", + "origin": "roboflow-train", + "isNasChild": false + } + ], + "trainings": [ + { + "id": "training-fast", + "projectId": "chess", + "versionId": "131", + "status": "finished", + "modelType": "rfdetr-small", + "modelIds": [ + "acme/chess-fast" + ] + }, + { + "id": "training-dominated", + "projectId": "chess", + "versionId": "131", + "status": "finished", + "modelType": "rfdetr-small", + "modelIds": [ + "acme/chess-dominated" + ] + }, + { + "id": "training-missing-latency", + "projectId": "chess", + "versionId": "131", + "status": "finished", + "modelType": "rfdetr-small", + "modelIds": [ + "acme/chess-missing-latency" + ] + } + ], + "evaluations": [ + { + "id": "eval-fast", + "modelId": "acme/chess-fast", + "project": "chess", + "versionId": "131", + "status": "done", + "createdAt": "2026-09-16T12:00:00.000Z", + "summary": { + "mAP": 0.8, + "mIoU": null, + "f1": null, + "precision": null, + "recall": null, + "medianLatencyMs": 3 + }, + "servingDevice": "T4" + }, + { + "id": "eval-dominated", + "modelId": "acme/chess-dominated", + "project": "chess", + "versionId": "131", + "status": "done", + "createdAt": "2026-09-16T12:00:00.000Z", + "summary": { + "mAP": 0.7, + "mIoU": null, + "f1": null, + "precision": null, + "recall": null, + "medianLatencyMs": 5 + }, + "servingDevice": "T4" + }, + { + "id": "eval-missing-latency", + "modelId": "acme/chess-missing-latency", + "project": "chess", + "versionId": "131", + "status": "done", + "createdAt": "2026-09-16T12:00:00.000Z", + "summary": { + "mAP": 0.9, + "mIoU": null, + "f1": null, + "precision": null, + "recall": null, + "medianLatencyMs": null + }, + "servingDevice": null + } + ], + "candidates": [ + { + "modelId": "acme/chess-fast", + "trainingId": "training-fast", + "evalId": "eval-fast", + "eligible": true, + "exclusionReason": null, + "accuracy": 0.8, + "medianLatencyMs": 3, + "onFrontier": true, + "currentAttempt": { + "evalId": "eval-fast", + "status": "done", + "createdAt": "2026-09-16T12:00:00.000Z" + }, + "rerun": { + "available": false, + "reason": null + } + }, + { + "modelId": "acme/chess-dominated", + "trainingId": "training-dominated", + "evalId": "eval-dominated", + "eligible": true, + "exclusionReason": null, + "accuracy": 0.7, + "medianLatencyMs": 5, + "onFrontier": false, + "currentAttempt": { + "evalId": "eval-dominated", + "status": "done", + "createdAt": "2026-09-16T12:00:00.000Z" + }, + "rerun": { + "available": false, + "reason": null + } + }, + { + "modelId": "acme/chess-missing-latency", + "trainingId": "training-missing-latency", + "evalId": "eval-missing-latency", + "eligible": false, + "exclusionReason": "latency_unavailable", + "accuracy": 0.9, + "medianLatencyMs": null, + "onFrontier": false, + "currentAttempt": { + "evalId": "eval-missing-latency", + "status": "done", + "createdAt": "2026-09-16T12:00:00.000Z" + }, + "rerun": { + "available": true, + "reason": "latency_unavailable" + } + } + ], + "appUrl": "https://app.roboflow.com/acme/chess/131" +} From aa67454367e5f2c8de3f719ea47ffb6b0d93cb71 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Wed, 16 Sep 2026 15:14:21 -0230 Subject: [PATCH 2/9] docs: link model comparison to unified reference --- CLI-COMMANDS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index c0f2f286..9b3635ac 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -370,7 +370,7 @@ view. Items left in Trash are cleaned up automatically after 30 days. Compare models on a Dataset Version with `roboflow eval compare --project chess-pieces --version 131`. Use `--json` for the full result, including server-computed frontier membership, exclusions, and current evaluation attempts. The command is read-only. It needs `model-eval:read` access and Model Comparison enabled for the Workspace. -See the [Model Comparison reference](https://docs.roboflow.com/train/model-comparison) for the response contract and all command options. +See the [Model Comparison reference](https://docs.roboflow.com/models/evaluate/model-comparison) for the response contract and all command options. ```bash # List evals in the workspace; filter by project, version, model, or status. From d114553d4683f288136687f8c89911505ddbcf16 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Mon, 21 Sep 2026 15:14:28 -0230 Subject: [PATCH 3/9] fix: use public model comparison contract --- CLI-COMMANDS.md | 7 +- roboflow/adapters/rfapi.py | 8 +- roboflow/cli/handlers/eval.py | 40 +++-- roboflow/core/workspace.py | 28 ++- tests/adapters/test_rfapi_model_evals.py | 19 +- tests/cli/test_eval_handler.py | 32 ++-- tests/fixtures/model_eval_comparison.json | 202 ++++------------------ tests/test_model_eval.py | 22 +++ 8 files changed, 144 insertions(+), 214 deletions(-) diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 9b3635ac..579b10d4 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -367,9 +367,10 @@ view. Items left in Trash are cleaned up automatically after 30 days. ### Inspect model evaluations -Compare models on a Dataset Version with `roboflow eval compare --project chess-pieces --version 131`. -Use `--json` for the full result, including server-computed frontier membership, exclusions, and current evaluation attempts. -The command is read-only. It needs `model-eval:read` access and Model Comparison enabled for the Workspace. +Compare models with `roboflow eval compare --project chess-pieces --version 131`. +Use `--frontier-metric mAP5095` to select a metric instead of the project default. +Use `--json` for all model metrics, median latency, exclusions, and server-computed frontier membership. +The command is read-only. It needs `model-eval:read` access. See the [Model Comparison reference](https://docs.roboflow.com/models/evaluate/model-comparison) for the response contract and all command options. ```bash diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index ec0adad2..2169fb3d 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -2184,8 +2184,14 @@ def compare_model_evals( *, project: str, version: Union[str, int], + frontier_metric: Optional[str] = None, ) -> dict: - return _eval_get(api_key, workspace_url, "/compare", params={"project": project, "version": version}) + return _eval_get( + api_key, + workspace_url, + "/compare", + params={"project": project, "version": version, "frontierMetric": frontier_metric}, + ) def list_model_evals( diff --git a/roboflow/cli/handlers/eval.py b/roboflow/cli/handlers/eval.py index ce234c57..d00fe8ba 100644 --- a/roboflow/cli/handlers/eval.py +++ b/roboflow/cli/handlers/eval.py @@ -29,56 +29,60 @@ def compare_evals_cmd( ctx: typer.Context, project: Annotated[str, typer.Option("-p", "--project", help="Project slug")], version: Annotated[int, typer.Option("-v", "--version", min=1, help="Dataset version number")], + frontier_metric: Annotated[ + Optional[str], + typer.Option("--frontier-metric", help="Metric for frontier membership (uses the project default if omitted)"), + ] = None, ) -> None: """Compare test-set accuracy and median latency without starting evaluations.""" from roboflow.adapters import rfapi from roboflow.cli._output import output, output_error from roboflow.cli._table import format_table - args = ctx_to_args(ctx, project=project, version=version) + args = ctx_to_args(ctx, project=project, version=version, frontier_metric=frontier_metric) resolved = _resolve(args) if not resolved: return workspace_url, api_key = resolved try: - comparison = rfapi.compare_model_evals(api_key, workspace_url, project=project, version=version) + comparison = rfapi.compare_model_evals( + api_key, + workspace_url, + project=project, + version=version, + frontier_metric=frontier_metric, + ) except Exception as exc: output_error( args, str(exc), - hint=( - "Check the project, version, and workspace access. Model Comparison must be enabled for the workspace." - ), + hint="Check the project, version, frontier metric, and workspace access.", exit_code=_eval_error_exit_code(exc), ) return if args.json: output(args, comparison) return - metric = comparison.get("metric") or {} - names = {model["id"]: model["name"] for model in comparison.get("models", [])} + frontier_metric = comparison.get("frontierMetric") rows = [] - for candidate in comparison.get("candidates", []): - accuracy = candidate.get("accuracy") - latency = candidate.get("medianLatencyMs") + for model in comparison.get("models", []): + accuracy = model.get("metrics", {}).get(frontier_metric) if frontier_metric else None + latency = model.get("medianLatencyMs") rows.append( { - "model": names.get(candidate["modelId"], candidate["modelId"]), + "model": model["modelId"], "accuracy": f"{accuracy * 100:.1f}%" if accuracy is not None else "", "latency": f"{latency:.2f}" if latency is not None else "", - "frontier": "Yes" if candidate.get("onFrontier") else "", - "exclusion": candidate.get("exclusionReason") or "", + "frontier": "Yes" if model.get("onFrontier") else "", + "exclusion": model.get("exclusionReason") or "", } ) table = format_table( rows, columns=["model", "accuracy", "latency", "frontier", "exclusion"], - headers=["MODEL", metric.get("label", "ACCURACY"), "MEDIAN LATENCY (ms)", "FRONTIER", "EXCLUSION"], + headers=["MODEL", frontier_metric or "ACCURACY", "MEDIAN LATENCY (ms)", "FRONTIER", "EXCLUSION"], ) - lines = [f"Test set | Serving device: {comparison.get('servingDevice', '')}", table] - if comparison.get("appUrl"): - lines.append(comparison["appUrl"]) - output(args, comparison, text="\n".join(lines)) + output(args, comparison, text=table) @eval_app.command("list") diff --git a/roboflow/core/workspace.py b/roboflow/core/workspace.py index 69fd055f..d4e3a634 100644 --- a/roboflow/core/workspace.py +++ b/roboflow/core/workspace.py @@ -8,7 +8,7 @@ import tempfile import time import zipfile -from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional +from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union import requests from requests.exceptions import HTTPError @@ -1572,6 +1572,32 @@ def upload_vision_event_image( # Model evaluations # ----------------------------------------------------------------- + def compare_model_evaluations( + self, + project: str, + version: Union[str, int], + *, + frontier_metric: Optional[str] = None, + ) -> dict: + """Compare model accuracy and median latency for a dataset version. + + Args: + project: Project URL slug. + version: Dataset version number. + frontier_metric: Metric for frontier membership. The server uses + the project default when this value is not specified. + + Returns: + The public model comparison response. + """ + return rfapi.compare_model_evals( + self.__api_key, + self.url, + project=project, + version=version, + frontier_metric=frontier_metric, + ) + def evals( self, *, diff --git a/tests/adapters/test_rfapi_model_evals.py b/tests/adapters/test_rfapi_model_evals.py index e0dac4f0..b758d906 100644 --- a/tests/adapters/test_rfapi_model_evals.py +++ b/tests/adapters/test_rfapi_model_evals.py @@ -222,24 +222,27 @@ def test_non_json_body_falls_back_to_text(self, mock_get): class TestCompareModelEvals(unittest.TestCase): @patch("roboflow.adapters.rfapi.requests.get") - def test_compare_returns_complete_server_result(self, mock_get): + def test_compare_returns_public_result_and_sends_frontier_metric(self, mock_get): comparison = { "project": "chess", - "version": {"id": "131", "projectId": "chess", "projectType": "object-detection"}, - "metric": {"key": "mAP", "label": "mAP@50", "unit": "ratio"}, - "candidates": [{"modelId": f"ws/chess-{index}", "onFrontier": False} for index in range(201)], - "evaluations": [], + "version": "131", + "frontierMetric": "mAP5095", + "availableMetrics": ["mAP", "mAP5095"], "models": [], - "trainings": [], } mock_get.return_value = _resp(200, comparison) - result = rfapi.compare_model_evals("k", "ws", project="chess", version="131") + result = rfapi.compare_model_evals("k", "ws", project="chess", version="131", frontier_metric="mAP5095") self.assertEqual(result, comparison) mock_get.assert_called_once_with( f"{API_URL}/ws/model-evals/compare", - params={"api_key": "k", "project": "chess", "version": "131"}, + params={ + "api_key": "k", + "project": "chess", + "version": "131", + "frontierMetric": "mAP5095", + }, ) diff --git a/tests/cli/test_eval_handler.py b/tests/cli/test_eval_handler.py index 11a50c19..45fda2bc 100644 --- a/tests/cli/test_eval_handler.py +++ b/tests/cli/test_eval_handler.py @@ -433,12 +433,15 @@ def test_json_preserves_the_full_comparison(self, mock_get): "chess", "--version", "131", + "--frontier-metric", + "mAP5095", ], ) self.assertEqual(result.exit_code, 0, result.output) self.assertEqual(json.loads(result.stdout), comparison) self.assertEqual(mock_get.call_count, 1) + self.assertEqual(mock_get.call_args.kwargs["params"]["frontierMetric"], "mAP5095") @patch("roboflow.adapters.rfapi.requests.get") def test_text_shows_server_frontier_and_exclusion_with_zero_values(self, mock_get): @@ -446,24 +449,28 @@ def test_text_shows_server_frontier_and_exclusion_with_zero_values(self, mock_ge mock_get.return_value = MagicMock(status_code=200) mock_get.return_value.json.return_value = { - "metric": {"key": "mAP", "label": "mAP@50", "unit": "ratio"}, - "servingDevice": "T4", + "project": "chess", + "version": "131", + "frontierMetric": "mAP", + "availableMetrics": ["mAP"], "models": [ - {"id": "ws/chess-fast", "name": "Fast model"}, - {"id": "ws/chess-old", "name": "Old model"}, - ], - "candidates": [ - {"modelId": "ws/chess-fast", "eligible": True, "accuracy": 0, "medianLatencyMs": 0, "onFrontier": True}, + { + "modelId": "ws/chess-fast", + "evaluationId": "eval-fast", + "metrics": {"mAP": 0}, + "medianLatencyMs": 0, + "onFrontier": True, + "exclusionReason": None, + }, { "modelId": "ws/chess-old", - "eligible": False, - "accuracy": 0.9, + "evaluationId": "eval-old", + "metrics": {"mAP": 0.9}, "medianLatencyMs": None, "onFrontier": False, "exclusionReason": "latency_unavailable", }, ], - "appUrl": "https://app.roboflow.com/ws/chess/131", } result = runner.invoke( app, @@ -473,16 +480,15 @@ def test_text_shows_server_frontier_and_exclusion_with_zero_values(self, mock_ge self.assertEqual(result.exit_code, 0, result.output) for text in [ "MODEL", - "mAP@50", + "mAP", "MEDIAN LATENCY (ms)", "FRONTIER", "EXCLUSION", - "Fast model", + "ws/chess-fast", "0.0%", "0.00", "Yes", "latency_unavailable", - "https://app.roboflow.com/ws/chess/131", ]: self.assertIn(text, result.stdout) diff --git a/tests/fixtures/model_eval_comparison.json b/tests/fixtures/model_eval_comparison.json index 9fcbd307..b5de5e11 100644 --- a/tests/fixtures/model_eval_comparison.json +++ b/tests/fixtures/model_eval_comparison.json @@ -1,194 +1,56 @@ { "project": "chess", - "version": { - "id": "131", - "projectId": "chess", - "projectType": "object-detection" - }, - "metric": { - "key": "mAP", - "label": "mAP@50", - "unit": "ratio" - }, - "servingDevice": "T4", + "version": "131", + "frontierMetric": "mAP", + "availableMetrics": ["mAP", "mAP5095", "mAP75", "precision", "recall", "f1"], "models": [ { - "id": "acme/chess-fast", - "name": "Fast model", - "url": "acme/chess-fast", - "projectId": "chess", - "versionId": "131", - "trainingId": "training-fast", - "modelType": "rfdetr-small", - "origin": "roboflow-train", - "isNasChild": false - }, - { - "id": "acme/chess-dominated", - "name": "Dominated model", - "url": "acme/chess-dominated", - "projectId": "chess", - "versionId": "131", - "trainingId": "training-dominated", - "modelType": "rfdetr-small", - "origin": "roboflow-train", - "isNasChild": false - }, - { - "id": "acme/chess-missing-latency", - "name": "Previously evaluated model", - "url": "acme/chess-missing-latency", - "projectId": "chess", - "versionId": "131", - "trainingId": "training-missing-latency", - "modelType": "rfdetr-small", - "origin": "roboflow-train", - "isNasChild": false - } - ], - "trainings": [ - { - "id": "training-fast", - "projectId": "chess", - "versionId": "131", - "status": "finished", - "modelType": "rfdetr-small", - "modelIds": [ - "acme/chess-fast" - ] - }, - { - "id": "training-dominated", - "projectId": "chess", - "versionId": "131", - "status": "finished", - "modelType": "rfdetr-small", - "modelIds": [ - "acme/chess-dominated" - ] - }, - { - "id": "training-missing-latency", - "projectId": "chess", - "versionId": "131", - "status": "finished", - "modelType": "rfdetr-small", - "modelIds": [ - "acme/chess-missing-latency" - ] - } - ], - "evaluations": [ - { - "id": "eval-fast", "modelId": "acme/chess-fast", - "project": "chess", - "versionId": "131", - "status": "done", - "createdAt": "2026-09-16T12:00:00.000Z", - "summary": { + "evaluationId": "eval-fast", + "metrics": { "mAP": 0.8, "mIoU": null, - "f1": null, - "precision": null, - "recall": null, - "medianLatencyMs": 3 + "f1": 0.8, + "precision": 0.8, + "recall": 0.8, + "mAP5095": 0.8, + "mAP75": 0.8 }, - "servingDevice": "T4" + "medianLatencyMs": 3, + "onFrontier": true, + "exclusionReason": null }, { - "id": "eval-dominated", "modelId": "acme/chess-dominated", - "project": "chess", - "versionId": "131", - "status": "done", - "createdAt": "2026-09-16T12:00:00.000Z", - "summary": { + "evaluationId": "eval-dominated", + "metrics": { "mAP": 0.7, "mIoU": null, - "f1": null, - "precision": null, - "recall": null, - "medianLatencyMs": 5 + "f1": 0.7, + "precision": 0.7, + "recall": 0.7, + "mAP5095": 0.7, + "mAP75": 0.7 }, - "servingDevice": "T4" + "medianLatencyMs": 5, + "onFrontier": false, + "exclusionReason": null }, { - "id": "eval-missing-latency", "modelId": "acme/chess-missing-latency", - "project": "chess", - "versionId": "131", - "status": "done", - "createdAt": "2026-09-16T12:00:00.000Z", - "summary": { + "evaluationId": "eval-missing-latency", + "metrics": { "mAP": 0.9, "mIoU": null, - "f1": null, - "precision": null, - "recall": null, - "medianLatencyMs": null + "f1": 0.9, + "precision": 0.9, + "recall": 0.9, + "mAP5095": 0.9, + "mAP75": 0.9 }, - "servingDevice": null - } - ], - "candidates": [ - { - "modelId": "acme/chess-fast", - "trainingId": "training-fast", - "evalId": "eval-fast", - "eligible": true, - "exclusionReason": null, - "accuracy": 0.8, - "medianLatencyMs": 3, - "onFrontier": true, - "currentAttempt": { - "evalId": "eval-fast", - "status": "done", - "createdAt": "2026-09-16T12:00:00.000Z" - }, - "rerun": { - "available": false, - "reason": null - } - }, - { - "modelId": "acme/chess-dominated", - "trainingId": "training-dominated", - "evalId": "eval-dominated", - "eligible": true, - "exclusionReason": null, - "accuracy": 0.7, - "medianLatencyMs": 5, - "onFrontier": false, - "currentAttempt": { - "evalId": "eval-dominated", - "status": "done", - "createdAt": "2026-09-16T12:00:00.000Z" - }, - "rerun": { - "available": false, - "reason": null - } - }, - { - "modelId": "acme/chess-missing-latency", - "trainingId": "training-missing-latency", - "evalId": "eval-missing-latency", - "eligible": false, - "exclusionReason": "latency_unavailable", - "accuracy": 0.9, "medianLatencyMs": null, "onFrontier": false, - "currentAttempt": { - "evalId": "eval-missing-latency", - "status": "done", - "createdAt": "2026-09-16T12:00:00.000Z" - }, - "rerun": { - "available": true, - "reason": "latency_unavailable" - } + "exclusionReason": "latency_unavailable" } - ], - "appUrl": "https://app.roboflow.com/acme/chess/131" + ] } diff --git a/tests/test_model_eval.py b/tests/test_model_eval.py index 40072bab..31c5fec6 100644 --- a/tests/test_model_eval.py +++ b/tests/test_model_eval.py @@ -246,6 +246,28 @@ def test_refresh_404_propagates(self, mock_fn): class TestWorkspaceEvalAccessors(unittest.TestCase): + @patch("roboflow.adapters.rfapi.compare_model_evals") + def test_compare_model_evaluations_returns_public_comparison(self, mock_compare): + comparison = { + "project": "chess", + "version": "131", + "frontierMetric": "mAP5095", + "availableMetrics": ["mAP", "mAP5095"], + "models": [], + } + mock_compare.return_value = comparison + + result = _make_workspace().compare_model_evaluations("chess", 131, frontier_metric="mAP5095") + + self.assertEqual(result, comparison) + mock_compare.assert_called_once_with( + "k", + "lee-sandbox", + project="chess", + version=131, + frontier_metric="mAP5095", + ) + @patch("roboflow.adapters.rfapi.list_model_evals") def test_evals_returns_modeleval_instances(self, mock_list): from roboflow.core.model_eval import ModelEval From c9048c932f274e17895c7b3f5daf8b1b640e654e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:45:20 +0000 Subject: [PATCH 4/9] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto=20?= =?UTF-8?q?format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/fixtures/model_eval_comparison.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/fixtures/model_eval_comparison.json b/tests/fixtures/model_eval_comparison.json index b5de5e11..909ea58e 100644 --- a/tests/fixtures/model_eval_comparison.json +++ b/tests/fixtures/model_eval_comparison.json @@ -2,7 +2,14 @@ "project": "chess", "version": "131", "frontierMetric": "mAP", - "availableMetrics": ["mAP", "mAP5095", "mAP75", "precision", "recall", "f1"], + "availableMetrics": [ + "mAP", + "mAP5095", + "mAP75", + "precision", + "recall", + "f1" + ], "models": [ { "modelId": "acme/chess-fast", From b7800127901dcfd028a6c9bcb800aa598103c481 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Tue, 22 Sep 2026 13:47:16 -0230 Subject: [PATCH 5/9] chore: bump version to 1.5.1 --- roboflow/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roboflow/__init__.py b/roboflow/__init__.py index 70f793eb..6c556500 100644 --- a/roboflow/__init__.py +++ b/roboflow/__init__.py @@ -21,7 +21,7 @@ CLIPModel = None # type: ignore[assignment,misc] GazeModel = None # type: ignore[assignment,misc] -__version__ = "1.5.0" +__version__ = "1.5.1" def check_key(api_key, model, notebook, num_retries=0): From fe359600b4b4693651db2e3945b0444abcff1f6b Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Tue, 22 Sep 2026 13:48:55 -0230 Subject: [PATCH 6/9] docs: add 1.5.1 changelog entry --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 756d3587..9b427346 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ All notable changes to this project will be documented in this file. +## 1.5.1 + +### Added + +- Model evaluation comparison for a dataset version + ([#529](https://github.com/roboflow/roboflow-python/pull/529)): + - `Workspace.compare_model_evaluations(project, version, frontier_metric=None)` + — compare test-set accuracy and median latency, including Pareto frontier + membership and reasons models are excluded. + - `roboflow --workspace eval compare --project --version ` + — display the comparison as a table; use `--json` for the public API response. + Use `--frontier-metric` to choose the metric for frontier membership. + - Both surfaces read existing results without starting evaluations. + ## 1.5.0 ### Added From e943941e4c5bec5c0aa569f3101ec5b5e92bf6d1 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Tue, 22 Sep 2026 13:57:33 -0230 Subject: [PATCH 7/9] docs: clarify model comparison and NAS starring commands --- CLI-COMMANDS.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 579b10d4..3c43294f 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -125,6 +125,7 @@ roboflow --json train results my-project/3 | jq -r .modelGroup roboflow model list -p my-project --group rfdetrNasGroup-3 # Star a NAS-trained model (triggers TRT compile for its recommended hardware): +# Also starts model evaluation when the workspace has Model Evaluation access. # --json train results … gives you the modelId per row. roboflow model star roboflow model star --unstar @@ -367,12 +368,6 @@ view. Items left in Trash are cleaned up automatically after 30 days. ### Inspect model evaluations -Compare models with `roboflow eval compare --project chess-pieces --version 131`. -Use `--frontier-metric mAP5095` to select a metric instead of the project default. -Use `--json` for all model metrics, median latency, exclusions, and server-computed frontier membership. -The command is read-only. It needs `model-eval:read` access. -See the [Model Comparison reference](https://docs.roboflow.com/models/evaluate/model-comparison) for the response contract and all command options. - ```bash # List evals in the workspace; filter by project, version, model, or status. roboflow eval list --status done --limit 10 @@ -396,6 +391,19 @@ without parsing message strings: `3` for `model_eval_not_found` (404), `invalid_split` / `invalid_confidence` (400). Requires the `model-eval:read` scope on the api key. +### Compare Models + +```bash +# Compare accuracy and latency for one version. +roboflow --workspace my-workspace eval compare --project my-project --version 3 + +# Choose the Pareto frontier metric; return all metrics and exclusion reasons. +roboflow eval compare --project my-project --version 3 --frontier-metric mAP5095 --json +``` + +Reads existing evaluations; does not start new ones. Requires `model-eval:read` +and workspace Model Evaluation access. + ### Workspace stats and billing ```bash From 8bdace0c2ad74a223596bd4e1a20cc7291983484 Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Tue, 22 Sep 2026 14:18:26 -0230 Subject: [PATCH 8/9] fix: clarify evaluation access errors and compare command conventions --- CHANGELOG.md | 5 ++ CLI-COMMANDS.md | 3 +- roboflow/adapters/rfapi.py | 1 + roboflow/cli/handlers/eval.py | 105 ++++++++++++++++++--------------- tests/cli/test_eval_handler.py | 60 ++++++++++++++++--- 5 files changed, 117 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b427346..b9f2bee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ All notable changes to this project will be documented in this file. Use `--frontier-metric` to choose the metric for frontier membership. - Both surfaces read existing results without starting evaluations. +### Changed + +- All `roboflow eval` commands now return exit code `2` for HTTP 401/403 + access errors (previously `1`). + ## 1.5.0 ### Added diff --git a/CLI-COMMANDS.md b/CLI-COMMANDS.md index 3c43294f..bd5c619e 100644 --- a/CLI-COMMANDS.md +++ b/CLI-COMMANDS.md @@ -386,7 +386,8 @@ roboflow eval recommendations --json ``` Exit codes are stable per error class so scripts and agents can react -without parsing message strings: `3` for `model_eval_not_found` (404), +without parsing message strings: `2` for authentication or access errors (401/403), +`3` for `model_eval_not_found` (404), `4` for `model_eval_not_done` (409 — eval still running), `5` for `invalid_split` / `invalid_confidence` (400). Requires the `model-eval:read` scope on the api key. diff --git a/roboflow/adapters/rfapi.py b/roboflow/adapters/rfapi.py index 2169fb3d..72f88701 100644 --- a/roboflow/adapters/rfapi.py +++ b/roboflow/adapters/rfapi.py @@ -2186,6 +2186,7 @@ def compare_model_evals( version: Union[str, int], frontier_metric: Optional[str] = None, ) -> dict: + """GET /{workspace}/model-evals/compare — compare models on a dataset version.""" return _eval_get( api_key, workspace_url, diff --git a/roboflow/cli/handlers/eval.py b/roboflow/cli/handlers/eval.py index d00fe8ba..f666bcb4 100644 --- a/roboflow/cli/handlers/eval.py +++ b/roboflow/cli/handlers/eval.py @@ -31,58 +31,15 @@ def compare_evals_cmd( version: Annotated[int, typer.Option("-v", "--version", min=1, help="Dataset version number")], frontier_metric: Annotated[ Optional[str], - typer.Option("--frontier-metric", help="Metric for frontier membership (uses the project default if omitted)"), + typer.Option( + "--frontier-metric", + help="Frontier metric: mAP, mAP5095, mAP75, mIoU, precision, recall, f1 (default: project default)", + ), ] = None, ) -> None: """Compare test-set accuracy and median latency without starting evaluations.""" - from roboflow.adapters import rfapi - from roboflow.cli._output import output, output_error - from roboflow.cli._table import format_table - args = ctx_to_args(ctx, project=project, version=version, frontier_metric=frontier_metric) - resolved = _resolve(args) - if not resolved: - return - workspace_url, api_key = resolved - try: - comparison = rfapi.compare_model_evals( - api_key, - workspace_url, - project=project, - version=version, - frontier_metric=frontier_metric, - ) - except Exception as exc: - output_error( - args, - str(exc), - hint="Check the project, version, frontier metric, and workspace access.", - exit_code=_eval_error_exit_code(exc), - ) - return - if args.json: - output(args, comparison) - return - frontier_metric = comparison.get("frontierMetric") - rows = [] - for model in comparison.get("models", []): - accuracy = model.get("metrics", {}).get(frontier_metric) if frontier_metric else None - latency = model.get("medianLatencyMs") - rows.append( - { - "model": model["modelId"], - "accuracy": f"{accuracy * 100:.1f}%" if accuracy is not None else "", - "latency": f"{latency:.2f}" if latency is not None else "", - "frontier": "Yes" if model.get("onFrontier") else "", - "exclusion": model.get("exclusionReason") or "", - } - ) - table = format_table( - rows, - columns=["model", "accuracy", "latency", "frontier", "exclusion"], - headers=["MODEL", frontier_metric or "ACCURACY", "MEDIAN LATENCY (ms)", "FRONTIER", "EXCLUSION"], - ) - output(args, comparison, text=table) + _compare_evals(args) @eval_app.command("list") @@ -247,6 +204,8 @@ def _hint_for(exc: Exception) -> Optional[str]: """Per-error actionable hint shown alongside the message in non-JSON mode.""" from roboflow.adapters import rfapi + if isinstance(exc, rfapi.ModelEvalAccessError): + return "Check your API key, its model-eval:read scope, and workspace Model Evaluation access." if isinstance(exc, rfapi.ModelEvalNotFoundError): return "Run 'roboflow eval list' to see eval ids in this workspace." if isinstance(exc, rfapi.ModelEvalNotDoneError): @@ -258,6 +217,56 @@ def _hint_for(exc: Exception) -> Optional[str]: return None +def _compare_evals(args): # noqa: ANN001 + from roboflow.adapters import rfapi + from roboflow.cli._output import output, output_error + from roboflow.cli._table import format_table + + resolved = _resolve(args) + if not resolved: + return + workspace_url, api_key = resolved + try: + comparison = rfapi.compare_model_evals( + api_key, + workspace_url, + project=args.project, + version=args.version, + frontier_metric=args.frontier_metric, + ) + except Exception as exc: + output_error( + args, + str(exc), + hint="Check the project, version, frontier metric, and workspace access.", + exit_code=_eval_error_exit_code(exc), + ) + return + if args.json: + output(args, comparison) + return + frontier_metric = comparison.get("frontierMetric") + rows = [] + for model in comparison.get("models", []): + accuracy = model.get("metrics", {}).get(frontier_metric) if frontier_metric else None + latency = model.get("medianLatencyMs") + rows.append( + { + "model": model["modelId"], + "accuracy": f"{accuracy * 100:.1f}%" if accuracy is not None else "", + "latency": f"{latency:.2f}" if latency is not None else "", + "frontier": "Yes" if model.get("onFrontier") else "", + "exclusion": model.get("exclusionReason") or "", + } + ) + table = format_table( + rows, + columns=["model", "accuracy", "latency", "frontier", "exclusion"], + headers=["MODEL", frontier_metric or "ACCURACY", "MEDIAN LATENCY (ms)", "FRONTIER", "EXCLUSION"], + ) + output(args, comparison, text=table) + + def _list_evals(args): # noqa: ANN001 from roboflow.adapters import rfapi from roboflow.cli._output import output, output_error diff --git a/tests/cli/test_eval_handler.py b/tests/cli/test_eval_handler.py index 45fda2bc..b326c630 100644 --- a/tests/cli/test_eval_handler.py +++ b/tests/cli/test_eval_handler.py @@ -5,7 +5,8 @@ import json import unittest from argparse import Namespace -from unittest.mock import patch +from pathlib import Path +from unittest.mock import MagicMock, patch from typer.testing import CliRunner @@ -397,6 +398,7 @@ def test_exit_codes(self) -> None: from roboflow.cli.handlers.eval import _eval_error_exit_code cases = { + rfapi.ModelEvalAccessError("x"): 2, rfapi.ModelEvalNotFoundError("x"): 3, rfapi.ModelEvalNotDoneError("x"): 4, rfapi.InvalidSplitError("x"): 5, @@ -411,10 +413,56 @@ def test_exit_codes(self) -> None: class TestEvalCompareCommand(unittest.TestCase): @patch("roboflow.adapters.rfapi.requests.get") - def test_json_preserves_the_full_comparison(self, mock_get): - from pathlib import Path - from unittest.mock import MagicMock + def test_not_found_is_a_structured_error(self, mock_get): + mock_get.return_value = MagicMock(status_code=404, text="Project not found") + mock_get.return_value.json.return_value = {"error": "Project not found"} + result = runner.invoke( + app, + ["--api-key", "k", "--workspace", "ws", "--json", "eval", "compare", "-p", "chess", "-v", "131"], + ) + self.assertEqual(result.exit_code, 3) + self.assertEqual(json.loads(result.stderr)["error"]["message"], "Project not found") + self.assertEqual(result.stdout, "") + + @patch("roboflow.adapters.rfapi.requests.get") + def test_null_frontier_metric_shows_exclusion_without_accuracy(self, mock_get): + mock_get.return_value = MagicMock(status_code=200) + mock_get.return_value.json.return_value = { + "frontierMetric": None, + "models": [ + { + "modelId": "ws/chess-model", + "metrics": {"mAP": 0.9}, + "medianLatencyMs": 10, + "onFrontier": False, + "exclusionReason": "unsupported_project_type", + } + ], + } + result = runner.invoke( + app, + ["--api-key", "k", "--workspace", "ws", "eval", "compare", "-p", "chess", "-v", "131"], + ) + + self.assertEqual(result.exit_code, 0, result.output) + for text in ["ACCURACY", "ws/chess-model", "10.00", "unsupported_project_type"]: + self.assertIn(text, result.stdout) + self.assertNotIn("90.0%", result.stdout) + + @patch("roboflow.adapters.rfapi.requests.get") + def test_list_access_error_includes_auth_and_entitlement_hint(self, mock_get): + mock_get.return_value = MagicMock(status_code=403, text="Access denied") + mock_get.return_value.json.return_value = {"error": "Access denied"} + result = runner.invoke(app, ["--api-key", "k", "--workspace", "ws", "--json", "eval", "list"]) + + self.assertEqual(result.exit_code, 2) + hint = json.loads(result.stderr)["error"]["hint"] + for text in ["API key", "model-eval:read", "Model Evaluation access"]: + self.assertIn(text, hint) + + @patch("roboflow.adapters.rfapi.requests.get") + def test_json_preserves_the_full_comparison(self, mock_get): comparison = json.loads((Path(__file__).parents[1] / "fixtures/model_eval_comparison.json").read_text()) mock_get.return_value = MagicMock(status_code=200) mock_get.return_value.json.return_value = comparison @@ -445,8 +493,6 @@ def test_json_preserves_the_full_comparison(self, mock_get): @patch("roboflow.adapters.rfapi.requests.get") def test_text_shows_server_frontier_and_exclusion_with_zero_values(self, mock_get): - from unittest.mock import MagicMock - mock_get.return_value = MagicMock(status_code=200) mock_get.return_value.json.return_value = { "project": "chess", @@ -494,8 +540,6 @@ def test_text_shows_server_frontier_and_exclusion_with_zero_values(self, mock_ge @patch("roboflow.adapters.rfapi.requests.get") def test_permission_failure_is_a_structured_auth_error(self, mock_get): - from unittest.mock import MagicMock - mock_get.return_value = MagicMock(status_code=403, text="Comparison access denied") mock_get.return_value.json.return_value = {"error": "forbidden", "message": "Comparison access denied"} result = runner.invoke( From 261b710347ecbaf66c7dff856589c682e4dbaa2f Mon Sep 17 00:00:00 2001 From: Lee Clement Date: Tue, 22 Sep 2026 14:25:29 -0230 Subject: [PATCH 9/9] fix: show access-specific hint for model comparison --- roboflow/cli/handlers/eval.py | 6 +++++- tests/cli/test_eval_handler.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/roboflow/cli/handlers/eval.py b/roboflow/cli/handlers/eval.py index f666bcb4..e55d556d 100644 --- a/roboflow/cli/handlers/eval.py +++ b/roboflow/cli/handlers/eval.py @@ -238,7 +238,11 @@ def _compare_evals(args): # noqa: ANN001 output_error( args, str(exc), - hint="Check the project, version, frontier metric, and workspace access.", + hint=( + _hint_for(exc) + if isinstance(exc, rfapi.ModelEvalAccessError) + else "Check the project, version, frontier metric, and workspace access." + ), exit_code=_eval_error_exit_code(exc), ) return diff --git a/tests/cli/test_eval_handler.py b/tests/cli/test_eval_handler.py index b326c630..4368d2ef 100644 --- a/tests/cli/test_eval_handler.py +++ b/tests/cli/test_eval_handler.py @@ -423,6 +423,10 @@ def test_not_found_is_a_structured_error(self, mock_get): self.assertEqual(result.exit_code, 3) self.assertEqual(json.loads(result.stderr)["error"]["message"], "Project not found") + self.assertEqual( + json.loads(result.stderr)["error"]["hint"], + "Check the project, version, frontier metric, and workspace access.", + ) self.assertEqual(result.stdout, "") @patch("roboflow.adapters.rfapi.requests.get") @@ -561,6 +565,9 @@ def test_permission_failure_is_a_structured_auth_error(self, mock_get): self.assertEqual(result.exit_code, 2) self.assertEqual(json.loads(result.stderr)["error"]["message"], "Comparison access denied") + hint = json.loads(result.stderr)["error"]["hint"] + for text in ["API key", "model-eval:read", "Model Evaluation access"]: + self.assertIn(text, hint) self.assertEqual(result.stdout, "")