diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 29148b75..4d194180 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -153,6 +153,7 @@ jobs: shell: bash run: | mise exec python@${{ matrix.python-version }} -- python ./py/scripts/nox-matrix.py ${{ matrix.shard }} 6 \ + --python-version '${{ matrix.python-version }}' --os '${{ matrix.os }}' \ --exclude-static-checks \ --exclude-session test_api_codegen diff --git a/py/nox_session_constraints.py b/py/nox_session_constraints.py new file mode 100644 index 00000000..2cde3d65 --- /dev/null +++ b/py/nox_session_constraints.py @@ -0,0 +1,98 @@ +"""Python and platform constraints shared by nox and its CI sharder.""" + +import re + + +CONSTRAINTS = { + "test_ai_sdk": {"min_python": "3.12"}, + "test_litellm": {"versions": {"latest": {"min_python": "3.11"}}}, + "test_livekit_agents": {"max_python_exclusive": "3.14", "ci_platforms": ["linux"]}, + "test_pipecat": {"min_python": "3.11", "max_python_exclusive": "3.14"}, + "test_agentscope": {"versions": {"latest": {"min_python": "3.11"}}}, + "test_dspy": {"versions": {"latest": {"min_python": "3.11"}}}, + "test_crewai": {"max_python_exclusive": "3.14"}, + "test_deepagents": {"min_python": "3.11"}, + "test_transformers": {"versions": {"not_latest": {"max_python_exclusive": "3.13"}}}, + "test_harbor": {"min_python": "3.12"}, + "test_otel": {"versions": {"not_latest_below": {"version": "1.28.0", "skip_from_python": "3.14"}}}, +} + + +def _version_key(value: str) -> tuple[int, ...]: + """Compare the numeric release versions used by this session matrix.""" + return tuple(int(component) for component in re.findall(r"\d+", value)) + + +def _normalize_platform(operating_system: str) -> str: + platform_name = operating_system.lower() + if platform_name.startswith("windows"): + return "windows" + if platform_name.startswith(("ubuntu", "linux")): + return "linux" + if platform_name.startswith(("macos", "darwin")): + return "darwin" + return platform_name + + +def _session_rules(session_id: str) -> tuple[dict, list[dict]]: + match = re.fullmatch(r"([^()]+)(?:\(([^()]*)\))?", session_id) + if not match: + return {}, [] + name, version = match.groups() + constraint = CONSTRAINTS.get(name) + if not constraint: + return {}, [] + + version_rules = constraint.get("versions", {}) + selected = [] + if version is not None: + if version == "latest": + selected.append(version_rules.get("latest", {})) + else: + selected.append(version_rules.get("not_latest", {})) + threshold = version_rules.get("not_latest_below") + if threshold and _version_key(version) < _version_key(threshold["version"]): + selected.append(threshold) + return constraint, selected + + +def incompatibility_reason( + session_id: str, + python_version: str, + operating_system: str, + *, + check_ci_platforms: bool = False, +) -> str | None: + """Return the declared reason a session is incompatible, if any.""" + python = _version_key(python_version) + platform_name = _normalize_platform(operating_system) + constraint, version_rules = _session_rules(session_id) + for rule in [constraint, *version_rules]: + if check_ci_platforms and "ci_platforms" in rule and platform_name not in rule["ci_platforms"]: + return f"not supported on {operating_system}" + if "min_python" in rule and python < _version_key(rule["min_python"]): + return f"requires Python {rule['min_python']}+" + if "max_python_exclusive" in rule and python >= _version_key(rule["max_python_exclusive"]): + return f"does not support Python {rule['max_python_exclusive']}+" + if "skip_from_python" in rule and python >= _version_key(rule["skip_from_python"]): + return f"skips on Python {rule['skip_from_python']}+" + return None + + +def session_is_compatible( + session_id: str, + python_version: str, + operating_system: str, + *, + check_ci_platforms: bool = False, +) -> bool: + """Return whether the requested nox session has work for this target.""" + return ( + incompatibility_reason( + session_id, + python_version, + operating_system, + check_ci_platforms=check_ci_platforms, + ) + is None + ) diff --git a/py/noxfile.py b/py/noxfile.py index 790c88cb..75f82f37 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -26,6 +26,11 @@ from packaging.version import Version +sys.path.insert(0, str(pathlib.Path(__file__).parent)) + +from nox_session_constraints import incompatibility_reason # noqa: E402 + + if sys.version_info >= (3, 11): import tomllib else: @@ -149,6 +154,13 @@ def _install_group_locked( os.unlink(req_file) +def _skip_if_incompatible(session: nox.Session) -> None: + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + reason = incompatibility_reason(session.name, python_version, platform.system()) + if reason: + session.skip(reason) + + def _get_matrix_versions(prefix: str) -> tuple[str, ...]: """Read the version matrix for *prefix* from ``[tool.braintrust.matrix]``. @@ -252,87 +264,86 @@ def _pinned_python_version(): AI_SDK_VERSIONS = _get_matrix_versions("ai-sdk") - -@nox.session() -@nox.parametrize("version", AI_SDK_VERSIONS, ids=AI_SDK_VERSIONS) -def test_ai_sdk(session, version): - """Test the Vercel AI SDK for Python telemetry integration.""" - if sys.version_info < (3, 12): - session.skip("Vercel AI SDK for Python requires Python 3.12+") - _install_test_deps(session) - _install_matrix_dep(session, "openai", LATEST) - _install_matrix_dep(session, "ai-sdk", version) - _run_tests(session, f"{INTEGRATION_DIR}/ai_sdk/test_ai_sdk.py", version=version) - - ANTHROPIC_VERSIONS = _get_matrix_versions("anthropic") - -@nox.session() -@nox.parametrize("version", ANTHROPIC_VERSIONS, ids=ANTHROPIC_VERSIONS) -def test_anthropic(session, version): - _install_test_deps(session) - _install_matrix_dep(session, "anthropic", version) - _run_tests(session, f"{INTEGRATION_DIR}/anthropic/test_anthropic.py", version=version) - - COHERE_VERSIONS = _get_matrix_versions("cohere") - -@nox.session() -@nox.parametrize("version", COHERE_VERSIONS, ids=COHERE_VERSIONS) -def test_cohere(session, version): - """Test the native Cohere SDK integration.""" - _install_test_deps(session) - _install_matrix_dep(session, "cohere", version) - _run_tests(session, f"{INTEGRATION_DIR}/cohere/test_cohere.py", version=version) - - BOTO3_VERSIONS = _get_matrix_versions("boto3") - -@nox.session() -@nox.parametrize("version", BOTO3_VERSIONS, ids=BOTO3_VERSIONS) -def test_bedrock_runtime(session, version): - """Test the boto3 Bedrock Runtime integration.""" - _install_test_deps(session) - _install_matrix_dep(session, "boto3", version) - _install_matrix_dep(session, "botocore", version) - _run_tests(session, f"{INTEGRATION_DIR}/bedrock_runtime/test_bedrock_runtime.py", version=version) - - INSTRUCTOR_VERSIONS = _get_matrix_versions("instructor") - -@nox.session() -@nox.parametrize("version", INSTRUCTOR_VERSIONS, ids=INSTRUCTOR_VERSIONS) -def test_instructor(session, version): - """Test the native Instructor structured-output integration.""" - _install_test_deps(session, "test-instructor") - _install_matrix_dep(session, "instructor", version) - # The locked auxiliary group selects an OpenAI SDK version supported by - # Instructor. These tests use VCR cassettes against ``api.openai.com``. - _run_tests(session, f"{INTEGRATION_DIR}/instructor/test_instructor.py", version=version) - - OPENAI_VERSIONS = _get_matrix_versions("openai") OPENAI_ENDPOINT_VERSIONS = (OPENAI_VERSIONS[0], OPENAI_VERSIONS[-1]) -@nox.session() -@nox.parametrize("version", OPENAI_VERSIONS, ids=OPENAI_VERSIONS) -def test_openai(session, version): - _install_test_deps(session) - _install_matrix_dep(session, "openai", version) - _run_tests( - session, - [ - f"{INTEGRATION_DIR}/openai/test_openai.py", - f"{INTEGRATION_DIR}/openai/test_oai_attachments.py", - f"{INTEGRATION_DIR}/openai/test_openai_openrouter_gateway.py", - ], - version=version, - ) +def _register_matrix_sessions(specs): + """Register the routine provider sessions from compact data records.""" + for spec in specs: + + def run(session, version, spec=spec): + _skip_if_incompatible(session) + _install_test_deps(session, *spec.get("groups", ())) + for package, package_version in spec["dependencies"]: + _install_matrix_dep(session, package, version if package_version == "$version" else package_version) + _run_tests( + session, + spec["tests"], + version=version, + env=spec.get("env"), + ) + + run.__name__ = spec["name"] + run.__qualname__ = spec["name"] + run.__doc__ = spec.get("doc") + decorated = nox.parametrize("version", spec["versions"], ids=spec["versions"])(run) + globals()[spec["name"]] = nox.session()(decorated) + + +_register_matrix_sessions( + [ + { + "name": "test_ai_sdk", + "versions": AI_SDK_VERSIONS, + "dependencies": [("openai", LATEST), ("ai-sdk", "$version")], + "tests": f"{INTEGRATION_DIR}/ai_sdk/test_ai_sdk.py", + }, + { + "name": "test_anthropic", + "versions": ANTHROPIC_VERSIONS, + "dependencies": [("anthropic", "$version")], + "tests": f"{INTEGRATION_DIR}/anthropic/test_anthropic.py", + }, + { + "name": "test_cohere", + "versions": COHERE_VERSIONS, + "dependencies": [("cohere", "$version")], + "tests": f"{INTEGRATION_DIR}/cohere/test_cohere.py", + }, + { + "name": "test_bedrock_runtime", + "versions": BOTO3_VERSIONS, + "dependencies": [("boto3", "$version"), ("botocore", "$version")], + "tests": f"{INTEGRATION_DIR}/bedrock_runtime/test_bedrock_runtime.py", + }, + { + "name": "test_instructor", + "versions": INSTRUCTOR_VERSIONS, + "groups": ("test-instructor",), + "dependencies": [("instructor", "$version")], + "tests": f"{INTEGRATION_DIR}/instructor/test_instructor.py", + }, + { + "name": "test_openai", + "versions": OPENAI_VERSIONS, + "dependencies": [("openai", "$version")], + "tests": [ + f"{INTEGRATION_DIR}/openai/test_openai.py", + f"{INTEGRATION_DIR}/openai/test_oai_attachments.py", + f"{INTEGRATION_DIR}/openai/test_openai_openrouter_gateway.py", + ], + }, + ] +) @nox.session() @@ -392,8 +403,7 @@ def test_openai_agents(session, version): @nox.parametrize("version", LITELLM_VERSIONS, ids=LITELLM_VERSIONS) def test_litellm(session, version): # LiteLLM 1.97.0 leaves Pydantic forward references unresolved on Python 3.10. - if version == LATEST and sys.version_info < (3, 11): - session.skip("Latest LiteLLM is currently broken on Python 3.10") + _skip_if_incompatible(session) _install_test_deps(session) # Auxiliary deps (openai upper-bounded, fastapi, orjson) are locked in the lockfile. _install_group_locked(session, "test-litellm") @@ -450,8 +460,7 @@ def test_agno(session, version): @nox.session() @nox.parametrize("version", LIVEKIT_AGENTS_VERSIONS, ids=LIVEKIT_AGENTS_VERSIONS) def test_livekit_agents(session, version): - if sys.version_info >= (3, 14): - session.skip("LiveKit Agents Silero VAD depends on onnxruntime, which does not ship Python 3.14 wheels") + _skip_if_incompatible(session) _install_test_deps(session) _install_matrix_dep(session, "livekit-agents", version) _install_group_locked(session, "test-livekit-agents") @@ -482,10 +491,7 @@ def test_livekit_agents(session, version): @nox.session() @nox.parametrize("version", PIPECAT_VERSIONS, ids=PIPECAT_VERSIONS) def test_pipecat(session, version): - if sys.version_info < (3, 11): - session.skip("Pipecat AI 1.x requires Python 3.11+") - if sys.version_info >= (3, 14): - session.skip("Pipecat AI's onnxruntime dependency does not ship Python 3.14 wheels") + _skip_if_incompatible(session) _install_test_deps(session) _install_group_locked(session, "test-pipecat") _install_matrix_dep(session, "pipecat-ai", version) @@ -517,8 +523,7 @@ def test_strands(session, version): @nox.session() @nox.parametrize("version", AGENTSCOPE_VERSIONS, ids=AGENTSCOPE_VERSIONS) def test_agentscope(session, version): - if version == LATEST and sys.version_info < (3, 11): - session.skip("AgentScope 2.x requires Python 3.11+") + _skip_if_incompatible(session) _install_test_deps(session) # AgentScope 1.0.0 imports streamablehttp_client, which MCP 2 no longer exports. constraint_group = "test-mcp-v1" if version == "1.0.0" else None @@ -621,8 +626,7 @@ def test_google_discoveryengine(session, version): def test_dspy(session, version): # DSPy latest preinstalls our latest LiteLLM pin, which is currently broken # on Python 3.10 due to unresolved Pydantic forward references. - if version == LATEST and sys.version_info < (3, 11): - session.skip("DSPy latest currently resolves a LiteLLM version broken on Python 3.10") + _skip_if_incompatible(session) _install_test_deps(session) if version == LATEST: # DSPy only lower-bounds LiteLLM, whose 1.92.0 release lacks Windows @@ -639,10 +643,7 @@ def test_dspy(session, version): @nox.session() @nox.parametrize("version", CREWAI_VERSIONS, ids=CREWAI_VERSIONS) def test_crewai(session, version): - if sys.version_info >= (3, 14): - session.skip( - "CrewAI currently resolves instructor -> pydantic-core builds that do not ship Python 3.14 wheels" - ) + _skip_if_incompatible(session) _install_test_deps(session) _install_group_locked(session, "test-crewai") _install_matrix_dep(session, "crewai", version) @@ -686,8 +687,7 @@ def test_langchain(session, version): @nox.session() @nox.parametrize("version", DEEPAGENTS_VERSIONS, ids=DEEPAGENTS_VERSIONS) def test_deepagents(session, version): - if sys.version_info < (3, 11): - session.skip("Deep Agents requires Python 3.11+") + _skip_if_incompatible(session) _install_test_deps(session) _install_group_locked(session, "test-deepagents") _install_matrix_dep(session, "deepagents", version) @@ -765,8 +765,7 @@ def test_huggingface_hub(session, version): def test_transformers(session, version): """Test local Hugging Face Transformers pipeline instrumentation.""" # The 4.42.0 floor pins tokenizers 0.19, whose wheels stop at Python 3.12. - if version != LATEST and sys.version_info >= (3, 13): - session.skip(f"Transformers {version} does not support Python 3.13+") + _skip_if_incompatible(session) _install_test_deps(session) _install_matrix_dep(session, "transformers", version) _install_group_locked(session, "test-transformers", indexes=("pytorch-cpu",)) @@ -795,8 +794,7 @@ def test_temporal(session, version): @nox.session() @nox.parametrize("version", HARBOR_VERSIONS, ids=HARBOR_VERSIONS) def test_harbor(session, version): - if sys.version_info < (3, 12): - session.skip("Harbor requires Python 3.12+") + _skip_if_incompatible(session) _install_test_deps(session) _install_matrix_dep(session, "harbor", version) _run_tests(session, f"{INTEGRATION_DIR}/harbor", version=version) @@ -865,8 +863,7 @@ def test_cli(session): @nox.parametrize("version", OTEL_VERSIONS, ids=OTEL_VERSIONS) def test_otel(session, version): """Test OtelExporter with OpenTelemetry installed.""" - if version != LATEST and Version(version) < Version("1.28.0") and sys.version_info >= (3, 14): - session.skip("OpenTelemetry <1.28 requires protobuf<5, which does not support Python 3.14") + _skip_if_incompatible(session) _install_test_deps(session) _install_matrix_dep(session, "opentelemetry-api", version) _install_matrix_dep(session, "opentelemetry-sdk", version) diff --git a/py/scripts/nox-matrix.py b/py/scripts/nox-matrix.py index 9bdf1854..abc38192 100644 --- a/py/scripts/nox-matrix.py +++ b/py/scripts/nox-matrix.py @@ -11,12 +11,17 @@ import argparse import json +import platform import re import subprocess import sys from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from nox_session_constraints import session_is_compatible # noqa: E402 + + # Sessions that run in the dedicated static_checks CI job and should be # excluded from the sharded nox matrix. Both checks.yaml and # update-session-weights.yaml reference this via --exclude-static-checks. @@ -81,11 +86,18 @@ def assign_shards( return shard_assignments +def filter_compatible_sessions(sessions: list[str], python_version: str, operating_system: str) -> list[str]: + """Remove sessions which are declared to skip for this CI target.""" + return [s for s in sessions if session_is_compatible(s, python_version, operating_system, check_ci_platforms=True)] + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("shard_index", type=int, help="Zero-based shard index") parser.add_argument("num_shards", type=int, help="Total number of shards") parser.add_argument("--dry-run", action="store_true", help="Print assignment without running nox") + parser.add_argument("--python-version", default=f"{sys.version_info.major}.{sys.version_info.minor}") + parser.add_argument("--os", dest="operating_system", default=platform.system()) parser.add_argument( "--exclude-session", action="append", @@ -132,6 +144,7 @@ def main() -> None: weights_file = root_dir / "py" / "scripts" / "session-weights.json" all_sessions = get_nox_sessions(noxfile) + all_sessions = filter_compatible_sessions(all_sessions, args.python_version, args.operating_system) excluded_sessions = set(args.exclude_session) if args.exclude_static_checks: excluded_sessions.update(STATIC_CHECK_SESSIONS)