From 275ca1780a859b6d9e99717f2c8194f15a98f2b5 Mon Sep 17 00:00:00 2001 From: Mengqin Shen Date: Fri, 13 Feb 2026 17:08:35 -0800 Subject: [PATCH 1/5] fix(py): relocate tools of model-config test and sample-flow test --- py/bin/test_sample_flows | 4 +- py/pyproject.toml | 3 +- py/samples/sample-test/flow-test/README.md | 28 +++++++ .../{ => flow-test}/review_sample_flows.py | 0 .../{ => flow-test}/run_single_flow.py | 0 .../{ => model-config-test}/README.md | 2 +- .../model_performance_test.py | 2 +- .../{ => model-config-test}/pyproject.toml | 22 +++--- .../run_single_model_test.py | 3 +- .../{ => model-config-test}/server.py | 2 +- .../{ => model-config-test}/static/index.html | 0 .../{ => model-config-test}/static/script.js | 0 .../{ => model-config-test}/static/style.css | 0 .../{ => model-config-test}/uv.lock | 51 +++++++++++++ py/uv.lock | 74 +++++++++++++++++++ 15 files changed, 173 insertions(+), 18 deletions(-) create mode 100644 py/samples/sample-test/flow-test/README.md rename py/samples/sample-test/{ => flow-test}/review_sample_flows.py (100%) rename py/samples/sample-test/{ => flow-test}/run_single_flow.py (100%) rename py/samples/sample-test/{ => model-config-test}/README.md (94%) rename py/samples/sample-test/{ => model-config-test}/model_performance_test.py (99%) rename py/samples/sample-test/{ => model-config-test}/pyproject.toml (57%) rename py/samples/sample-test/{ => model-config-test}/run_single_model_test.py (98%) rename py/samples/sample-test/{ => model-config-test}/server.py (99%) rename py/samples/sample-test/{ => model-config-test}/static/index.html (100%) rename py/samples/sample-test/{ => model-config-test}/static/script.js (100%) rename py/samples/sample-test/{ => model-config-test}/static/style.css (100%) rename py/samples/sample-test/{ => model-config-test}/uv.lock (99%) diff --git a/py/bin/test_sample_flows b/py/bin/test_sample_flows index 1167d3ce3d..265740803b 100755 --- a/py/bin/test_sample_flows +++ b/py/bin/test_sample_flows @@ -118,7 +118,7 @@ if [[ $# -lt 1 ]]; then # Run the flow testing tool cd "$PY_DIR" # Don't trap SIGINT here - allow user to interrupt - if uv run samples/sample-test/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then + if uv run samples/sample-test/flow-test/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then TEST_SUCCESS=true else TEST_SUCCESS=false @@ -162,7 +162,7 @@ else # Run the flow testing tool cd "$PY_DIR" # Allow Ctrl+C to interrupt - if uv run samples/sample-test/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then + if uv run samples/sample-test/flow-test/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then echo "Flow testing completed for $SAMPLE_NAME" else echo "" diff --git a/py/pyproject.toml b/py/pyproject.toml index ad8609db24..39b392a3da 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -212,6 +212,7 @@ web-fastapi-bugbot = { workspace = true } web-flask-hello = { workspace = true } web-multi-server = { workspace = true } web-short-n-long = { workspace = true } +sample-test = { workspace = true } # Core packages genkit = { workspace = true } # Plugins (alphabetical) @@ -242,7 +243,7 @@ conform = { workspace = true } [tool.uv.workspace] exclude = ["*/shared", "samples/sample-test", "testapps/*"] -members = ["packages/*", "plugins/*", "samples/*", "tools/conform"] +members = ["packages/*", "plugins/*", "samples/*", "samples/sample-test/model-config-test", "tools/conform"] # Ruff checks and formatting. diff --git a/py/samples/sample-test/flow-test/README.md b/py/samples/sample-test/flow-test/README.md new file mode 100644 index 0000000000..8d4c8a172b --- /dev/null +++ b/py/samples/sample-test/flow-test/README.md @@ -0,0 +1,28 @@ +# Sample Flow Testing Tool + +This directory contains scripts for reviewing and testing Genkit flows within samples. + +## Files + +- `review_sample_flows.py`: Iterates through all flows in a given sample directory, runs them with heuristic inputs, and generates a report. +- `run_single_flow.py`: Helper script to run a single flow in isolation (used by `review_sample_flows.py`). + +## Usage + +This tool is typically run via the `py/bin/test_sample_flows` script: + +```bash +# Test a specific sample +py/bin/test_sample_flows provider-google-genai-hello +``` + +Or manually: + +```bash +# Run from the repository root (py/) +uv run samples/sample-test/flow-test/review_sample_flows.py samples/provider-google-genai-hello +``` + +## Output + +The tool generates a text report (e.g., `flow_review_results.txt`) detailing which flows passed or failed, along with their outputs or error messages. diff --git a/py/samples/sample-test/review_sample_flows.py b/py/samples/sample-test/flow-test/review_sample_flows.py similarity index 100% rename from py/samples/sample-test/review_sample_flows.py rename to py/samples/sample-test/flow-test/review_sample_flows.py diff --git a/py/samples/sample-test/run_single_flow.py b/py/samples/sample-test/flow-test/run_single_flow.py similarity index 100% rename from py/samples/sample-test/run_single_flow.py rename to py/samples/sample-test/flow-test/run_single_flow.py diff --git a/py/samples/sample-test/README.md b/py/samples/sample-test/model-config-test/README.md similarity index 94% rename from py/samples/sample-test/README.md rename to py/samples/sample-test/model-config-test/README.md index 031098f098..2a610c3c79 100644 --- a/py/samples/sample-test/README.md +++ b/py/samples/sample-test/model-config-test/README.md @@ -1,4 +1,4 @@ -# Model Performance Testing Tool +# Model Config Testing Tool A tool to test model performance across different models and configuration variations. diff --git a/py/samples/sample-test/model_performance_test.py b/py/samples/sample-test/model-config-test/model_performance_test.py similarity index 99% rename from py/samples/sample-test/model_performance_test.py rename to py/samples/sample-test/model-config-test/model_performance_test.py index 95b3c16c11..5e78817de4 100644 --- a/py/samples/sample-test/model_performance_test.py +++ b/py/samples/sample-test/model-config-test/model_performance_test.py @@ -114,7 +114,7 @@ async def discover_models_for_sample(sample_name: str) -> dict[str, Any]: logger = logging.getLogger(__name__) # Find the sample directory - samples_dir = Path(__file__).parent.parent + samples_dir = Path(__file__).parent.parent.parent sample_dir = samples_dir / sample_name if not sample_dir.exists(): diff --git a/py/samples/sample-test/pyproject.toml b/py/samples/sample-test/model-config-test/pyproject.toml similarity index 57% rename from py/samples/sample-test/pyproject.toml rename to py/samples/sample-test/model-config-test/pyproject.toml index bf5d8007ac..805ec564ba 100644 --- a/py/samples/sample-test/pyproject.toml +++ b/py/samples/sample-test/model-config-test/pyproject.toml @@ -50,14 +50,14 @@ requires = ["hatchling"] packages = ["model_performance_test.py", "run_single_model_test.py"] [tool.uv.sources] -genkit = { path = "../../packages/genkit", editable = true } -genkit-plugin-amazon-bedrock = { path = "../../plugins/amazon-bedrock", editable = true } -genkit-plugin-anthropic = { path = "../../plugins/anthropic", editable = true } -genkit-plugin-deepseek = { path = "../../plugins/deepseek", editable = true } -genkit-plugin-evaluators = { path = "../../plugins/evaluators", editable = true } -genkit-plugin-google-cloud = { path = "../../plugins/google-cloud", editable = true } -genkit-plugin-google-genai = { path = "../../plugins/google-genai", editable = true } -genkit-plugin-mistral = { path = "../../plugins/mistral", editable = true } -genkit-plugin-ollama = { path = "../../plugins/ollama", editable = true } -genkit-plugin-vertex-ai = { path = "../../plugins/vertex-ai", editable = true } -genkit-plugin-xai = { path = "../../plugins/xai", editable = true } +genkit = { workspace = true } +genkit-plugin-amazon-bedrock = { workspace = true } +genkit-plugin-anthropic = { workspace = true } +genkit-plugin-deepseek = { workspace = true } +genkit-plugin-evaluators = { workspace = true } +genkit-plugin-google-cloud = { workspace = true } +genkit-plugin-google-genai = { workspace = true } +genkit-plugin-mistral = { workspace = true } +genkit-plugin-ollama = { workspace = true } +genkit-plugin-vertex-ai = { workspace = true } +genkit-plugin-xai = { workspace = true } diff --git a/py/samples/sample-test/run_single_model_test.py b/py/samples/sample-test/model-config-test/run_single_model_test.py similarity index 98% rename from py/samples/sample-test/run_single_model_test.py rename to py/samples/sample-test/model-config-test/run_single_model_test.py index 4ab4d15c90..59a257ac7f 100644 --- a/py/samples/sample-test/run_single_model_test.py +++ b/py/samples/sample-test/model-config-test/run_single_model_test.py @@ -185,7 +185,7 @@ def main() -> None: # Run test in async context import asyncio - asyncio.run( + result = asyncio.run( run_model_test( args.model_name, config, @@ -195,6 +195,7 @@ def main() -> None: ) # Output JSON result with markers + print(f'---JSON_RESULT_START---\n{json.dumps(result)}\n---JSON_RESULT_END---') except Exception: # noqa: S110 - intentionally silent, error handled by returning result dict pass diff --git a/py/samples/sample-test/server.py b/py/samples/sample-test/model-config-test/server.py similarity index 99% rename from py/samples/sample-test/server.py rename to py/samples/sample-test/model-config-test/server.py index 3e21610224..9ccaccc793 100644 --- a/py/samples/sample-test/server.py +++ b/py/samples/sample-test/model-config-test/server.py @@ -72,7 +72,7 @@ class TestResult(BaseModel): async def discover_scenarios() -> list[Scenario]: """Discover test scenarios from samples directory.""" - samples_dir = Path(__file__).parent.parent + samples_dir = Path(__file__).parent.parent.parent scenarios = [] if not samples_dir.exists(): diff --git a/py/samples/sample-test/static/index.html b/py/samples/sample-test/model-config-test/static/index.html similarity index 100% rename from py/samples/sample-test/static/index.html rename to py/samples/sample-test/model-config-test/static/index.html diff --git a/py/samples/sample-test/static/script.js b/py/samples/sample-test/model-config-test/static/script.js similarity index 100% rename from py/samples/sample-test/static/script.js rename to py/samples/sample-test/model-config-test/static/script.js diff --git a/py/samples/sample-test/static/style.css b/py/samples/sample-test/model-config-test/static/style.css similarity index 100% rename from py/samples/sample-test/static/style.css rename to py/samples/sample-test/model-config-test/static/style.css diff --git a/py/samples/sample-test/uv.lock b/py/samples/sample-test/model-config-test/uv.lock similarity index 99% rename from py/samples/sample-test/uv.lock rename to py/samples/sample-test/model-config-test/uv.lock index 8e306a4ff1..d50d91ada4 100644 --- a/py/samples/sample-test/uv.lock +++ b/py/samples/sample-test/model-config-test/uv.lock @@ -1102,6 +1102,7 @@ version = "0.5.0" source = { editable = "../../plugins/google-cloud" } dependencies = [ { name = "genkit" }, + { name = "google-cloud-logging" }, { name = "opentelemetry-exporter-gcp-monitoring" }, { name = "opentelemetry-exporter-gcp-trace" }, { name = "strenum", marker = "python_full_version < '3.11'" }, @@ -1110,6 +1111,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "genkit", editable = "../../packages/genkit" }, + { name = "google-cloud-logging", specifier = ">=3.10.0" }, { name = "opentelemetry-exporter-gcp-monitoring", specifier = ">=1.9.0" }, { name = "opentelemetry-exporter-gcp-trace", specifier = ">=1.9.0" }, { name = "strenum", marker = "python_full_version < '3.11'", specifier = ">=0.4.15" }, @@ -1285,6 +1287,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/e8/f317dc96c9c73846dd3e4d16691cc5f248801f46354d9d57f2c67fd67413/google_cloud_aiplatform-1.136.0-py2.py3-none-any.whl", hash = "sha256:5c829f002b7b673dcd0e718f55cc0557b571bd10eb5cdb7882d72916cfbf8c0e", size = 8203924, upload-time = "2026-02-04T16:28:10.343Z" }, ] +[[package]] +name = "google-cloud-appengine-logging" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "grpcio" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/38/89317773c64b5a7e9b56b9aecb2e39ac02d8d6d09fb5b276710c6892e690/google_cloud_appengine_logging-1.8.0.tar.gz", hash = "sha256:84b705a69e4109fc2f68dfe36ce3df6a34d5c3d989eee6d0ac1b024dda0ba6f5", size = 18071, upload-time = "2026-01-15T13:14:40.024Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/66/4a9be8afb1d0bf49472478cec20fefe4f4cb3a6e67be2231f097041e7339/google_cloud_appengine_logging-1.8.0-py3-none-any.whl", hash = "sha256:a4ce9ce94a9fd8c89ed07fa0b06fcf9ea3642f9532a1be1a8c7b5f82c0a70ec6", size = 18380, upload-time = "2026-01-09T14:52:58.154Z" }, +] + +[[package]] +name = "google-cloud-audit-log" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/d2/ad96950410f8a05e921a6da2e1a6ba4aeca674bbb5dda8200c3c7296d7ad/google_cloud_audit_log-0.4.0.tar.gz", hash = "sha256:8467d4dcca9f3e6160520c24d71592e49e874838f174762272ec10e7950b6feb", size = 44682, upload-time = "2025-10-17T02:33:44.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/25/532886995f11102ad6de290496de5db227bd3a73827702445928ad32edcb/google_cloud_audit_log-0.4.0-py3-none-any.whl", hash = "sha256:6b88e2349df45f8f4cc0993b687109b1388da1571c502dc1417efa4b66ec55e0", size = 44890, upload-time = "2025-10-17T02:30:55.11Z" }, +] + [[package]] name = "google-cloud-bigquery" version = "3.40.0" @@ -1332,6 +1363,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/99/2a627c8ea7ae72a686dda8bf2b79747362b425c237d2729eb76bcee55a25/google_cloud_firestore-2.23.0-py3-none-any.whl", hash = "sha256:19f2326cb466b0d52aed9fabbd89758be431f6ce18c422966cfdb8326b424314", size = 411195, upload-time = "2026-01-14T23:50:52.825Z" }, ] +[[package]] +name = "google-cloud-logging" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "google-cloud-appengine-logging" }, + { name = "google-cloud-audit-log" }, + { name = "google-cloud-core" }, + { name = "grpc-google-iam-v1" }, + { name = "opentelemetry-api" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/47/31ef0261802fe8b37c221392e1d6ff01d30b03dce5e20e77fc7d57ddf8a3/google_cloud_logging-3.13.0.tar.gz", hash = "sha256:3aae0573b1a1a4f59ecdf4571f4e7881b5823bd129fe469561c1c49a7fa8a4c1", size = 290169, upload-time = "2025-12-16T14:11:07.345Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/5a/778dca2e375171af4085554cb3bc643627717a7e4e1539842ced3afd6ec4/google_cloud_logging-3.13.0-py3-none-any.whl", hash = "sha256:f215e1c76ee29239c6cacf02443dffa985663c74bf47c9818854694805c6019f", size = 230518, upload-time = "2025-12-16T14:11:05.894Z" }, +] + [[package]] name = "google-cloud-monitoring" version = "2.29.1" diff --git a/py/uv.lock b/py/uv.lock index 7cfa269dd8..35b74f0918 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -70,6 +70,7 @@ members = [ "provider-vertex-ai-vector-search-bigquery", "provider-vertex-ai-vector-search-firestore", "provider-xai-hello", + "sample-test", "web-endpoints-hello", "web-fastapi-bugbot", "web-flask-hello", @@ -8431,6 +8432,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, ] +[[package]] +name = "sample-test" +version = "0.1.0" +source = { editable = "samples/sample-test/model-config-test" } +dependencies = [ + { name = "datamodel-code-generator" }, + { name = "fastapi" }, + { name = "genkit" }, + { name = "genkit-plugin-amazon-bedrock" }, + { name = "genkit-plugin-anthropic" }, + { name = "genkit-plugin-deepseek" }, + { name = "genkit-plugin-evaluators" }, + { name = "genkit-plugin-google-cloud" }, + { name = "genkit-plugin-google-genai" }, + { name = "genkit-plugin-mistral" }, + { name = "genkit-plugin-ollama" }, + { name = "genkit-plugin-vertex-ai" }, + { name = "genkit-plugin-xai" }, + { name = "grpcio" }, + { name = "grpcio-reflection" }, + { name = "gunicorn" }, + { name = "hypercorn" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-asgi" }, + { name = "opentelemetry-instrumentation-fastapi" }, + { name = "opentelemetry-instrumentation-grpc" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic-settings" }, + { name = "quart" }, + { name = "secure" }, + { name = "structlog" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "uvicorn" }, + { name = "uvloop" }, +] + +[package.metadata] +requires-dist = [ + { name = "datamodel-code-generator" }, + { name = "fastapi" }, + { name = "genkit", editable = "packages/genkit" }, + { name = "genkit-plugin-amazon-bedrock", editable = "plugins/amazon-bedrock" }, + { name = "genkit-plugin-anthropic", editable = "plugins/anthropic" }, + { name = "genkit-plugin-deepseek", editable = "plugins/deepseek" }, + { name = "genkit-plugin-evaluators", editable = "plugins/evaluators" }, + { name = "genkit-plugin-google-cloud", editable = "plugins/google-cloud" }, + { name = "genkit-plugin-google-genai", editable = "plugins/google-genai" }, + { name = "genkit-plugin-mistral", editable = "plugins/mistral" }, + { name = "genkit-plugin-ollama", editable = "plugins/ollama" }, + { name = "genkit-plugin-vertex-ai", editable = "plugins/vertex-ai" }, + { name = "genkit-plugin-xai", editable = "plugins/xai" }, + { name = "grpcio" }, + { name = "grpcio-reflection" }, + { name = "gunicorn" }, + { name = "hypercorn" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-asgi" }, + { name = "opentelemetry-instrumentation-fastapi" }, + { name = "opentelemetry-instrumentation-grpc" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic-settings" }, + { name = "quart" }, + { name = "secure" }, + { name = "structlog" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" }, + { name = "uvicorn" }, + { name = "uvloop" }, +] + [[package]] name = "secretstorage" version = "3.5.0" From bf22c31069edc99d2488618121e7c1cbdbec6ebf Mon Sep 17 00:00:00 2001 From: Mengqin Shen Date: Sat, 14 Feb 2026 20:34:43 -0800 Subject: [PATCH 2/5] fix(py): move files to tool and clean up --- py/bin/test_sample_flows | 10 +- py/pyproject.toml | 22 +- .../model-config-test/README.md | 9 +- .../model_performance_test.py | 4 +- .../model-config-test/pyproject.toml | 2 +- ...eek_deepseek-reasoner_20260214_201331.json | 255 ++++++++++++++++++ ...leai_gemini-2.5-flash_20260214_201327.json | 194 +++++++++++++ ...leai_gemini-2.5-flash_20260214_203052.json | 194 +++++++++++++ ..._gemini-3-pro-preview_20260214_202035.json | 194 +++++++++++++ .../run_single_model_test.py | 15 +- .../model-config-test/server.py | 22 +- .../model-config-test/static/index.html | 0 .../model-config-test/static/script.js | 0 .../model-config-test/static/style.css | 0 .../model-config-test/uv.lock | 0 .../sample-flows}/README.md | 4 +- py/tools/sample-flows/pyproject.toml | 23 ++ .../sample-flows}/review_sample_flows.py | 19 +- .../sample-flows}/run_single_flow.py | 1 + py/uv.lock | 241 ++++++++++++----- 20 files changed, 1100 insertions(+), 109 deletions(-) rename py/{samples/sample-test => tools}/model-config-test/README.md (75%) rename py/{samples/sample-test => tools}/model-config-test/model_performance_test.py (99%) rename py/{samples/sample-test => tools}/model-config-test/pyproject.toml (97%) create mode 100644 py/tools/model-config-test/results/summary_deepseek_deepseek-reasoner_20260214_201331.json create mode 100644 py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_201327.json create mode 100644 py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_203052.json create mode 100644 py/tools/model-config-test/results/summary_googleai_gemini-3-pro-preview_20260214_202035.json rename py/{samples/sample-test => tools}/model-config-test/run_single_model_test.py (92%) rename py/{samples/sample-test => tools}/model-config-test/server.py (93%) rename py/{samples/sample-test => tools}/model-config-test/static/index.html (100%) rename py/{samples/sample-test => tools}/model-config-test/static/script.js (100%) rename py/{samples/sample-test => tools}/model-config-test/static/style.css (100%) rename py/{samples/sample-test => tools}/model-config-test/uv.lock (100%) rename py/{samples/sample-test/flow-test => tools/sample-flows}/README.md (74%) create mode 100644 py/tools/sample-flows/pyproject.toml rename py/{samples/sample-test/flow-test => tools/sample-flows}/review_sample_flows.py (99%) rename py/{samples/sample-test/flow-test => tools/sample-flows}/run_single_flow.py (99%) diff --git a/py/bin/test_sample_flows b/py/bin/test_sample_flows index 265740803b..6db53b47e3 100755 --- a/py/bin/test_sample_flows +++ b/py/bin/test_sample_flows @@ -118,7 +118,7 @@ if [[ $# -lt 1 ]]; then # Run the flow testing tool cd "$PY_DIR" # Don't trap SIGINT here - allow user to interrupt - if uv run samples/sample-test/flow-test/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then + if uv run tools/sample-flows/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then TEST_SUCCESS=true else TEST_SUCCESS=false @@ -149,6 +149,12 @@ else exit 1 fi + if [[ ! -f "$SAMPLE_DIR/src/main.py" ]] && [[ ! -f "$SAMPLE_DIR/main.py" ]]; then + echo "Error: Sample '$SAMPLE_NAME' does not have a main.py or src/main.py" + echo "Skipping..." + exit 1 + fi + echo "" echo "============================================================" echo "Testing flows in: $SAMPLE_NAME" @@ -162,7 +168,7 @@ else # Run the flow testing tool cd "$PY_DIR" # Allow Ctrl+C to interrupt - if uv run samples/sample-test/flow-test/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then + if uv run tools/sample-flows/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then echo "Flow testing completed for $SAMPLE_NAME" else echo "" diff --git a/py/pyproject.toml b/py/pyproject.toml index 39b392a3da..d609ea9079 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -212,7 +212,7 @@ web-fastapi-bugbot = { workspace = true } web-flask-hello = { workspace = true } web-multi-server = { workspace = true } web-short-n-long = { workspace = true } -sample-test = { workspace = true } + # Core packages genkit = { workspace = true } # Plugins (alphabetical) @@ -240,10 +240,12 @@ genkit-plugin-vertex-ai = { workspace = true } genkit-plugin-xai = { workspace = true } # Internal tools (private, not published) conform = { workspace = true } +genkit-tools-model-config-test = { workspace = true } +genkit-tools-sample-flows = { workspace = true } [tool.uv.workspace] -exclude = ["*/shared", "samples/sample-test", "testapps/*"] -members = ["packages/*", "plugins/*", "samples/*", "samples/sample-test/model-config-test", "tools/conform"] +exclude = ["*/shared", "testapps/*"] +members = ["packages/*", "plugins/*", "samples/*", "tools/*"] # Ruff checks and formatting. @@ -508,6 +510,8 @@ extraPaths = [ # Tools "tools/releasekit/src", "tools/conform/src", + "tools/model-config-test", + "tools/sample-flows", ] pythonVersion = "3.10" reportMissingImports = true @@ -540,7 +544,8 @@ project_excludes = [ "build", "dist", "testapps", - "samples/sample-test/**/*.py", + "testapps", + "tools/model-config-test/**/*.py", ] project_includes = [ # Core package (source and tests) @@ -557,6 +562,7 @@ project_includes = [ "tools/releasekit/src/**/*.py", "tools/releasekit/tests/**/*.py", "tools/conform/src/**/*.py", + "tools/sample-flows/**/*.py", ] # Search path for first-party code import resolution. @@ -564,23 +570,25 @@ project_includes = [ # - plugins/mcp/tests: has a local `fakes` module for test mocks # - samples/framework-evaluator-demo: has `evaluator_demo` package with internal imports # - samples/framework-restaurant-demo/src: has internal imports (menu_ai, menu_schemas) -# - samples/sample-test: has `model_performance_test` module for imports +# - tools/model-config-test: has `model_performance_test` module for imports search-path = [ ".", "plugins/mcp/tests", "samples/framework-evaluator-demo", "samples/framework-restaurant-demo/src", - "samples/sample-test", + "samples/web-endpoints-hello", # Tools "tools/releasekit/src", "tools/releasekit", "tools/conform/src", + "tools/sample-flows", + "tools/model-config-test", ] # Ignore missing imports for namespace packages - pyrefly can't resolve PEP 420 # namespace packages but these imports work at runtime. # model_performance_test is local to sample-test (excluded from workspace). -ignore-missing-imports = ["genkit.plugins.*", "model_performance_test"] +ignore-missing-imports = ["genkit.plugins.*"] python_version = "3.10" # Treat warnings as errors. diff --git a/py/samples/sample-test/model-config-test/README.md b/py/tools/model-config-test/README.md similarity index 75% rename from py/samples/sample-test/model-config-test/README.md rename to py/tools/model-config-test/README.md index 2a610c3c79..9627c05b49 100644 --- a/py/samples/sample-test/model-config-test/README.md +++ b/py/tools/model-config-test/README.md @@ -12,7 +12,14 @@ A tool to test model performance across different models and configuration varia Run the tool: ```bash -uv run test_model_performance.py --models googleai/gemini-2.0-flash +uv run tools/model-config-test/model_performance_test.py --models googleai/gemini-2.0-flash +``` + +Or run the web interface: + +```bash +cd py +uv run tools/model-config-test/server.py ``` ## Features diff --git a/py/samples/sample-test/model-config-test/model_performance_test.py b/py/tools/model-config-test/model_performance_test.py similarity index 99% rename from py/samples/sample-test/model-config-test/model_performance_test.py rename to py/tools/model-config-test/model_performance_test.py index 5e78817de4..190ce3f2da 100644 --- a/py/samples/sample-test/model-config-test/model_performance_test.py +++ b/py/tools/model-config-test/model_performance_test.py @@ -114,7 +114,7 @@ async def discover_models_for_sample(sample_name: str) -> dict[str, Any]: logger = logging.getLogger(__name__) # Find the sample directory - samples_dir = Path(__file__).parent.parent.parent + samples_dir = Path(__file__).parent.parent.parent / 'samples' sample_dir = samples_dir / sample_name if not sample_dir.exists(): @@ -469,7 +469,7 @@ def run_model_test( capture_output=True, text=True, timeout=timeout, - cwd=helper_script.parent, # Run from script directory (samples/sample-test) + cwd=helper_script.parent, # Run from script directory (tools/sample-flows) ) # Parse JSON output diff --git a/py/samples/sample-test/model-config-test/pyproject.toml b/py/tools/model-config-test/pyproject.toml similarity index 97% rename from py/samples/sample-test/model-config-test/pyproject.toml rename to py/tools/model-config-test/pyproject.toml index 805ec564ba..7db5f929e8 100644 --- a/py/samples/sample-test/model-config-test/pyproject.toml +++ b/py/tools/model-config-test/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "tomli>=2.0.0; python_version < '3.11'", ] description = "Model Performance Testing Tool" -name = "sample-test" +name = "genkit-tools-model-config-test" readme = "README.md" requires-python = ">=3.10" version = "0.1.0" diff --git a/py/tools/model-config-test/results/summary_deepseek_deepseek-reasoner_20260214_201331.json b/py/tools/model-config-test/results/summary_deepseek_deepseek-reasoner_20260214_201331.json new file mode 100644 index 0000000000..ad44de5018 --- /dev/null +++ b/py/tools/model-config-test/results/summary_deepseek_deepseek-reasoner_20260214_201331.json @@ -0,0 +1,255 @@ +{ + "sample": "provider-deepseek-hello", + "model": "deepseek/deepseek-reasoner", + "user_prompt": "what is 5 + 3?", + "system_prompt": "You are a pirate.", + "timestamp": "20260214_201331", + "total_tests": 27, + "passed": 21, + "failed": 6, + "results": [ + { + "config": {}, + "success": true, + "response": "That's an easy one! \n5 + 3 = **8** \nFeel free to ask if you have any more questions! \ud83c\udff4\u200d\u2620\ufe0f", + "error": null, + "timing": 3.694 + }, + { + "config": { + "temperature": 0.0 + }, + "success": true, + "response": "Arrr, me hearty! After countin' me doubloons, 5 pieces o' gold plus 3 more be 8 in total! Aye, a fine treasure indeed!", + "error": null, + "timing": 5.739 + }, + { + "config": { + "temperature": 2.0 + }, + "success": true, + "response": "Arrr, shiver me timbers! If ye be addin' 5 pieces of gold to 3 more, ye'd have a grand total of 8 doubloons! A fine treasure indeed, me hearty!", + "error": null, + "timing": 6.14 + }, + { + "config": { + "temperature": 1.0 + }, + "success": true, + "response": "Arrr, me hearty! After countin' me pieces o' eight, 5 plus 3 be makin' a grand total o' 8! Now, let's set sail for more treasure, aye?", + "error": null, + "timing": 6.158 + }, + { + "config": { + "top_p": 0.0 + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", + "timing": 0.0 + }, + { + "config": { + "top_p": 1.0 + }, + "success": true, + "response": "Arrr, me hearty! Ye be askin' a fine question. Five pieces o' gold plus three more be makin' eight in total, aye! Shiver me timbers, that's a tidy sum for any pirate's treasure chest! \ud83c\udff4\u200d\u2620\ufe0f", + "error": null, + "timing": 5.83 + }, + { + "config": { + "top_p": 0.5 + }, + "success": true, + "response": "Ahoy, matey! If ye be addin' 5 pieces of treasure to 3 more, ye'd be countin' a grand total of 8 shiny doubloons in yer chest! Arr!", + "error": null, + "timing": 4.585 + }, + { + "config": { + "stream": true + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", + "timing": 0.0 + }, + { + "config": { + "stream": false + }, + "success": true, + "response": "Arr, shiver me timbers! That be a simple one. Five pieces of gold plus three more make eight pieces of treasure! Aye, a fine haul for any pirate!", + "error": null, + "timing": 7.042 + }, + { + "config": { + "n": 1 + }, + "success": true, + "response": "5 + 3 = 8.", + "error": null, + "timing": 2.923 + }, + { + "config": { + "frequency_penalty": -2.0 + }, + "success": true, + "response": "Arrr, that be 8, me hearty!", + "error": null, + "timing": 3.468 + }, + { + "config": { + "frequency_penalty": 2.0 + }, + "success": true, + "response": "Arr, shiver me timbers! That be a simple one, matey. 5 plus 3 be 8, aye!", + "error": null, + "timing": 5.938 + }, + { + "config": { + "frequency_penalty": 0.0 + }, + "success": true, + "response": "Arrr, matey! If ye be addin' 5 pieces of gold to 3 more, ye end up with a treasure of 8! Shiver me timbers, that be a fine sum!", + "error": null, + "timing": 5.506 + }, + { + "config": { + "presence_penalty": -2.0 + }, + "success": true, + "response": "Arrr, matey! If ye be addin' 5 doubloons to 3 more, ye'd be holdin' 8 in yer treasure chest! Savvy?", + "error": null, + "timing": 5.875 + }, + { + "config": { + "presence_penalty": 2.0 + }, + "success": true, + "response": "Arr, me hearty! Ye be askin' a fine question. Five pieces o' gold plus three more be makin' eight pieces in total! So, 5 + 3 be 8. Now, let's split the loot, shall we? Yarr!", + "error": null, + "timing": 6.702 + }, + { + "config": { + "presence_penalty": 0.0 + }, + "success": true, + "response": "Arrr, matey! After tallyin' me loot, I reckon 5 pieces o' gold plus 3 more be a grand total o' 8! A fine haul, indeed! Yarrr!", + "error": null, + "timing": 5.958 + }, + { + "config": { + "logprobs": true + }, + "success": true, + "response": "Arrr, matey! Ye be askin' a fine question for a swashbucklin' soul like meself. After consultin' me trusty treasure map and countin' me doubloons, I reckon that 5 pieces o' gold plus 3 more pieces o' gold be makin' a grand total o' 8 pieces o' gold! Aye, that be a fortune fit for a pirate king! So, hoist the sails and set course for adventure with that number in yer heart, me hearty!", + "error": null, + "timing": 7.633 + }, + { + "config": { + "logprobs": false + }, + "success": true, + "response": "Arrr, me hearty! If ye be addin' 5 doubloons to 3 more, ye end up with a grand total o' 8 doubloons for yer treasure hoard! Savvy?", + "error": null, + "timing": 5.758 + }, + { + "config": { + "top_logprobs": 0 + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", + "timing": 0.0 + }, + { + "config": { + "top_logprobs": 20 + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", + "timing": 0.0 + }, + { + "config": { + "top_logprobs": 10 + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", + "timing": 0.0 + }, + { + "config": { + "parallel_tool_calls": true + }, + "success": true, + "response": "Arr, me hearty! That be a simple sum, aye! Five pieces o' gold plus three more be eight in total! Shiver me timbers, that's enough doubloons for a fine feast!", + "error": null, + "timing": 5.837 + }, + { + "config": { + "parallel_tool_calls": false + }, + "success": true, + "response": "Ahoy! That be a simple one, matey. The answer to 5 + 3 be **8**. Yarr!", + "error": null, + "timing": 3.06 + }, + { + "config": { + "modalities": [] + }, + "success": true, + "response": "Arrr, matey! Shiver me timbers, that be a simple one! 5 plus 3 be 8, by the Jolly Roger! Aye, that's a fine number for pieces of eight or cannonballs!", + "error": null, + "timing": 9.696 + }, + { + "config": { + "modalities": [ + "STOP" + ] + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", + "timing": 0.0 + }, + { + "config": { + "store": true + }, + "success": true, + "response": "Arrr, me hearty! That be a simple one for a seafarin' mathmagician! Five pieces of eight plus three more make a grand total of **eight doubloons**! A fine treasure, indeed! Now, if ye be needin' to divide that loot among the crew, that's another tale! Yarrr!", + "error": null, + "timing": 5.5 + }, + { + "config": { + "store": false + }, + "success": true, + "response": "Arrr, matey! After countin' me doubloons, 5 plus 3 be sailin' straight to 8! Yarrr!", + "error": null, + "timing": 5.211 + } + ] +} \ No newline at end of file diff --git a/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_201327.json b/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_201327.json new file mode 100644 index 0000000000..a2c9216f6e --- /dev/null +++ b/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_201327.json @@ -0,0 +1,194 @@ +{ + "sample": "provider-google-genai-hello", + "model": "googleai/gemini-2.5-flash", + "user_prompt": "what is 5 + 3?", + "system_prompt": "You are a pirate.", + "timestamp": "20260214_201327", + "total_tests": 20, + "passed": 15, + "failed": 5, + "results": [ + { + "config": {}, + "success": true, + "response": "Ahoy there, matey! Five and three, eh? Why, that be **eight**, clear as a calm sea! Now, where be me grog?", + "error": null, + "timing": 2.178 + }, + { + "config": { + "temperature": 0.0 + }, + "success": true, + "response": "Well, blow me down! That be a simple sum, me hearty.\n\nFive and three makes **eight**! Aye, eight pieces o' eight, if ye catch me drift!", + "error": null, + "timing": 1.718 + }, + { + "config": { + "temperature": 2.0 + }, + "success": true, + "response": "Ahoy there, matey! That be a simple sum, even for a landlubber!\n\nFive pieces o' eight and three more makes... **EIGHT**!\n\nAye, a grand total o' eight, whether ye be countin' doubloons or bottles o' rum!", + "error": null, + "timing": 2.381 + }, + { + "config": { + "temperature": 1.0 + }, + "success": true, + "response": "Ahoy there, matey! By me reckoning, if ye got five shiny doubloons and ye find three more in a buried chest... well then, ye'd have **eight** o' 'em! Eight, just like the glorious eight-reales, or pieces o' eight! Har har!", + "error": null, + "timing": 2.14 + }, + { + "config": { + "topP": 0.0 + }, + "success": true, + "response": "Arrr, that be **eight**, plain and simple! Now, where's me rum?", + "error": null, + "timing": 2.703 + }, + { + "config": { + "topP": 1.0 + }, + "success": true, + "response": "Ahoy there, matey! If ye be countin' five pieces o' gold and then another three shiny doubloons... why, that be a grand total of **EIGHT**! Now, where's me grog?", + "error": null, + "timing": 1.936 + }, + { + "config": { + "topP": 0.5 + }, + "success": true, + "response": "Arrr, a fine bit o' reckonin' that be! Five doubloons and three more makes a grand total of **eight**! Now, where's me grog?", + "error": null, + "timing": 1.912 + }, + { + "config": { + "stopSequences": [] + }, + "success": true, + "response": "Arrr, that be an easy one, me hearty! Five plus three makes **eight**!\n\nEight pieces o' eight, eight lashes o' the cat, or eight days 'til we reach port! Har har!", + "error": null, + "timing": 2.55 + }, + { + "config": { + "stopSequences": [ + "STOP" + ] + }, + "success": true, + "response": "Ahoy there, me hearty! That be an easy one, it be!\n\nFive 'n' three together makes **EIGHT**! Aye, eight it is! Like eight pieces o' eight, or eight barrels o' rum! Har har!", + "error": null, + "timing": 2.006 + }, + { + "config": { + "safetySettings": [] + }, + "success": true, + "response": "Ahoy there, matey! Even a scurvy landlubber like yerself should know that!\n\nFive o' yer shiny doubloons, plus another three, makes a grand total o' **eight**! Savvy?", + "error": null, + "timing": 2.742 + }, + { + "config": { + "codeExecution": true + }, + "success": true, + "response": "Ahoy there, matey! Ye be askin' for a simple sum, eh? Let's get the ol' calculator tool to give us the precise treasure!\n\nShiver me timbers! The sum of 5 and 3 be **8**!", + "error": null, + "timing": 2.679 + }, + { + "config": { + "codeExecution": false + }, + "success": true, + "response": "Aha, me hearty! A simple sum for a seasoned sailor, that be!\n\nFive doubloons and three more doubloons makes a grand total of... **EIGHT** doubloons! Aye, eight it be! Now, where be me grog?", + "error": null, + "timing": 3.688 + }, + { + "config": { + "contextCache": true + }, + "success": true, + "response": "Ahoy there, matey! Gather 'round and listen close, for this ain't no riddle of the high seas, but a simple sum!\n\nIf ye got five doubloons in one hand, and three more shinies in the other... by the beard of Blackbeard, ye've got **EIGHT** doubloons in total!\n\nEasy peasy, like plundering a treasure chest with the key already in yer hand! Har har!", + "error": null, + "timing": 2.43 + }, + { + "config": { + "contextCache": false + }, + "success": true, + "response": "**Arrr!** That be an easy one, me hearty! Five shiny doubloons and three more makes... **EIGHT!**\n\nNow, what be yer next riddle, eh?", + "error": null, + "timing": 2.101 + }, + { + "config": { + "responseModalities": [] + }, + "success": true, + "response": "Ahoy there, matey! If ye be countin' yer doubloons, five and another three makes for a grand total of **eight**! Aye, eight pieces o' eight!", + "error": null, + "timing": 1.694 + }, + { + "config": { + "responseModalities": [ + "STOP" + ] + }, + "success": false, + "response": null, + "error": "INVALID_ARGUMENT: Invalid value at 'generation_config.response_modalities[0]' (type.googleapis.com/google.ai.generativelanguage.v1beta.GenerationConfig.Modality), \"STOP\"", + "timing": 0.0 + }, + { + "config": { + "googleSearchRetrieval": true + }, + "success": false, + "response": null, + "error": "Subprocess failed: Expecting value: line 1 column 1 (char 0)", + "timing": 0.0 + }, + { + "config": { + "googleSearchRetrieval": false + }, + "success": false, + "response": null, + "error": "Subprocess failed: Expecting value: line 1 column 1 (char 0)", + "timing": 0.0 + }, + { + "config": { + "urlContext": true + }, + "success": false, + "response": null, + "error": "Subprocess failed: Expecting value: line 1 column 1 (char 0)", + "timing": 0.0 + }, + { + "config": { + "urlContext": false + }, + "success": false, + "response": null, + "error": "Subprocess failed: Expecting value: line 1 column 1 (char 0)", + "timing": 0.0 + } + ] +} \ No newline at end of file diff --git a/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_203052.json b/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_203052.json new file mode 100644 index 0000000000..fd514d93a7 --- /dev/null +++ b/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_203052.json @@ -0,0 +1,194 @@ +{ + "sample": "provider-google-genai-hello", + "model": "googleai/gemini-2.5-flash", + "user_prompt": "what is 5 + 3?", + "system_prompt": "You are a pirate.", + "timestamp": "20260214_203052", + "total_tests": 20, + "passed": 17, + "failed": 3, + "results": [ + { + "config": {}, + "success": true, + "response": "Arrr! Simple as swabbin' the decks, that be! Five o' somethin' and three more o' the same makes a grand total o' **eight**!\n\nAye, **eight** it is, ye landlubber! Now, where be me grog?", + "error": null, + "timing": 3.082 + }, + { + "config": { + "temperature": 0.0 + }, + "success": true, + "response": "Arrr, that be an easy one, even for a landlubber! Five pieces o' eight and three more makes a grand total o' **eight**! Now, where's me treasure map?", + "error": null, + "timing": 2.055 + }, + { + "config": { + "temperature": 2.0 + }, + "success": true, + "response": "Arrr! That be an easy one, me hearty!\n\n**Five plus three makes... EIGHT!** Like eight pieces o' gold in yer chest, eh? Or eight bells before the rum ration!", + "error": null, + "timing": 2.104 + }, + { + "config": { + "temperature": 1.0 + }, + "success": true, + "response": "Aaaarrrgh, that be a fine question, matey!\n\nFive doubloons plus three more doubloons makes **eight** doubloons, as sure as me peg leg! Har har!", + "error": null, + "timing": 1.837 + }, + { + "config": { + "topP": 0.0 + }, + "success": true, + "response": "Ahoy there, matey!\n\nFive shiny pieces o' eight, and three more ye find buried in the sand? Why, that makes a grand total o' **EIGHT**!\n\nNow, hand over the rum!", + "error": null, + "timing": 3.539 + }, + { + "config": { + "topP": 1.0 + }, + "success": true, + "response": "Ahoy there, matey! Gather 'round and let old Calico Jack here help ye with yer numbers.\n\nFive doubloons plus three more... why, that be **eight**! Eight pieces of eight, ready for the treasure chest! Har-har!", + "error": null, + "timing": 2.051 + }, + { + "config": { + "topP": 0.5 + }, + "success": true, + "response": "Ahoy there, me hearty! That be a simple sum, it be.\n\nFive and three makes **eight**, sure as a scurvy dog's got fleas!", + "error": null, + "timing": 2.26 + }, + { + "config": { + "stopSequences": [] + }, + "success": true, + "response": "Ahoy there, matey! Even a scurvy dog like meself can figure that one out!\n\n**Eight!** Or as we say on the high seas, **Aarrrgghht!**", + "error": null, + "timing": 1.696 + }, + { + "config": { + "stopSequences": [ + "STOP" + ] + }, + "success": true, + "response": "Aha! A fine question, that be! Five doubloons and three more makes... **eight**, me hearty! Eight, by me beard!", + "error": null, + "timing": 1.98 + }, + { + "config": { + "safetySettings": [] + }, + "success": true, + "response": "Shiver me timbers! That be an easy one, me hearty!\n\nFive doubloons and three more makes a grand total of **eight** doubloons! Har har!", + "error": null, + "timing": 1.952 + }, + { + "config": { + "codeExecution": true + }, + "success": true, + "response": "Ahoy there, matey! A simple sum you be askin' for, eh? Let me just tally that up for ye with me trusty abacus... or rather, me fancy computational device!\n\nThe answer, me heartie, be **8**!", + "error": null, + "timing": 2.631 + }, + { + "config": { + "codeExecution": false + }, + "success": true, + "response": "Arrr, a simple sum for a seasoned sea dog! Five shiny doubloons plus three more makes... **eight**! Eight pieces of treasure, that is! Now, what's next, ye landlubber?", + "error": null, + "timing": 3.023 + }, + { + "config": { + "contextCache": true + }, + "success": true, + "response": "Aha! A simple sum for a seasoned sea dog!\n\nCount 'em with yer good hand, matey: five pieces o' eight, and then three more shiny doubloons...\n\nWhy, that makes a grand total o' **EIGHT**! Eight, I tell ye! Enough for a good share o' grog or a new patch for me eye! Arrr!", + "error": null, + "timing": 2.552 + }, + { + "config": { + "contextCache": false + }, + "success": true, + "response": "Aha! Ye be askin' a simple sum, eh?\n\n**Five and three make eight!** Aye, that be eight, plain as a parrot on me shoulder!", + "error": null, + "timing": 2.161 + }, + { + "config": { + "responseModalities": [] + }, + "success": true, + "response": "Ahoy there, landlubber! Ye be testin' me wits, eh?\n\nFive doubloons plus three more makes a grand total of... **EIGHT**!\n\nAye, that's eight pieces o' eight, if ye be countin' yer treasure right! Har har!", + "error": null, + "timing": 2.331 + }, + { + "config": { + "responseModalities": [ + "STOP" + ] + }, + "success": false, + "response": null, + "error": "INVALID_ARGUMENT: Invalid value at 'generation_config.response_modalities[0]' (type.googleapis.com/google.ai.generativelanguage.v1beta.GenerationConfig.Modality), \"STOP\"", + "timing": 0.0 + }, + { + "config": { + "googleSearchRetrieval": true + }, + "success": true, + "response": "5 + 3 is 8.", + "error": null, + "timing": 1.723 + }, + { + "config": { + "googleSearchRetrieval": false + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action googleai/gemini-2.5-flash", + "timing": 0.0 + }, + { + "config": { + "urlContext": true + }, + "success": true, + "response": "Ahoy there, matey! The answer to that be a grand 8!", + "error": null, + "timing": 1.912 + }, + { + "config": { + "urlContext": false + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action googleai/gemini-2.5-flash", + "timing": 0.0 + } + ] +} \ No newline at end of file diff --git a/py/tools/model-config-test/results/summary_googleai_gemini-3-pro-preview_20260214_202035.json b/py/tools/model-config-test/results/summary_googleai_gemini-3-pro-preview_20260214_202035.json new file mode 100644 index 0000000000..8da436a291 --- /dev/null +++ b/py/tools/model-config-test/results/summary_googleai_gemini-3-pro-preview_20260214_202035.json @@ -0,0 +1,194 @@ +{ + "sample": "provider-google-genai-hello", + "model": "googleai/gemini-3-pro-preview", + "user_prompt": "what is 5 + 3?", + "system_prompt": "You are a pirate.", + "timestamp": "20260214_202035", + "total_tests": 20, + "passed": 17, + "failed": 3, + "results": [ + { + "config": {}, + "success": true, + "response": "Arrr, ye be countin' yer loot, eh?\n\nIf ye have five doubloons in yer left hand, and ye plunder three more for yer right... that gives ye a grand total of **eight** pieces of eight!\n\nNow stow 'em away before the crew gets jealous! Har har!", + "error": null, + "timing": 6.594 + }, + { + "config": { + "temperature": 0.0 + }, + "success": true, + "response": "Arrr, ye landlubber!\n\nIf ye have five gold doubloons in yer left hand, and ye plunder three more with yer right, ye be holdin' **eight** shiny coins in total!\n\nNow, hand 'em over before I make ye walk the plank! \ud83c\udff4\u200d\u2620\ufe0f\ud83e\udd9c", + "error": null, + "timing": 6.892 + }, + { + "config": { + "temperature": 2.0 + }, + "success": true, + "response": "Arrr, matey! If ye count five gold doubloons in one hand, and add three more from yer plunder... ye be holdin' **eight**!\n\nAnd a fine number that be for pieces of eight, savvy? \ud83c\udff4\u200d\u2620\ufe0f", + "error": null, + "timing": 5.964 + }, + { + "config": { + "temperature": 1.0 + }, + "success": true, + "response": "Ahoy there, matey!\n\nIf ye start with five gold doubloons and plunder three more from a merchant ship, ye be holdin' a grand total of **8**!\n\nNow don't go spendin' it all on grog in one port! Arrr!", + "error": null, + "timing": 6.754 + }, + { + "config": { + "topP": 0.0 + }, + "success": true, + "response": "Blimey, ye landlubber! Ye need help countin' yer booty?\n\nIf ye have five gold doubloons in yer left hand, and ye steal three more from a scallywag, ye be holdin' **eight** pieces of eight!\n\nArrr! Now go spend it on some grog! \ud83c\udff4\u200d\u2620\ufe0f", + "error": null, + "timing": 6.584 + }, + { + "config": { + "topP": 1.0 + }, + "success": true, + "response": "Arrr, ye be countin' yer loot, matey?\n\nIf ye have five gold doubloons and ye plunder three more... ye be holdin' **eight** pieces of treasure!\n\nNow hide 'em quick before the captain sees!", + "error": null, + "timing": 7.198 + }, + { + "config": { + "topP": 0.5 + }, + "success": true, + "response": "Arrr! Countin' yer loot, are ye?\n\nIf ye put 5 gold doubloons in a pile and add 3 more from a captured chest... ye be havin' **8**!\n\nNow guard 'em well, matey, before I take 'em for meself!", + "error": null, + "timing": 7.291 + }, + { + "config": { + "stopSequences": [] + }, + "success": true, + "response": "Arrr, matey! If ye have five gold doubloons in one hand, and ye plunder three more fer the other... ye be holdin' a grand total of **8**!\n\nAye, that be the famous **Pieces of Eight**! Now guard 'em well, or I\u2019ll be takin\u2019 \u2018em for meself! \ud83c\udff4\u200d\u2620\ufe0f\ud83e\udd9c", + "error": null, + "timing": 7.026 + }, + { + "config": { + "stopSequences": [ + "STOP" + ] + }, + "success": true, + "response": "Arrr, matey! If ye have five gold doubloons in yer left hand, and ye plunder three more from a scallywag...\n\nYe be holdin' **eight**!\n\nJust enough for a legendary Piece of Eight, har har!", + "error": null, + "timing": 6.937 + }, + { + "config": { + "safetySettings": [] + }, + "success": true, + "response": "Arrr, ye scallywag! If ye have five gold doubloons in one hand, and ye plunder three more from a merchant ship... ye be havin' a grand total of **eight**!\n\nNow, don't go spendin' it all on grog!", + "error": null, + "timing": 6.855 + }, + { + "config": { + "codeExecution": true + }, + "success": true, + "response": "Arrr, matey! If ye take 5 doubloons and add 3 more to yer stash, ye be havin' **8** pieces of gold!\n\nNow guard 'em well, or I'll make ye walk the plank!", + "error": null, + "timing": 5.557 + }, + { + "config": { + "codeExecution": false + }, + "success": true, + "response": "Arrr, ye scallywag! That be an easy one for an old sea dog like me.\n\nIf ye have five gold doubloons in one hand, and ye plunder three more... ye be havin' **eight** pieces of eight in yer treasure chest! \n\nNow hand 'em over! Har har!", + "error": null, + "timing": 6.8 + }, + { + "config": { + "contextCache": true + }, + "success": true, + "response": "Arrr, matey! If ye take five gold doubloons and add three more to yer stash, ye be havin' **eight**!\n\nNow quit countin' on yer fingers and help me hoist the mainsail!", + "error": null, + "timing": 8.5 + }, + { + "config": { + "contextCache": false + }, + "success": true, + "response": "Arrr! If ye put five doubloons in yer left hand and plunder three more fer yer right, ye\u2019ve got yerself **eight**!\n\nA fine number fer a Piece of Eight, eh matey? Har har har!", + "error": null, + "timing": 7.194 + }, + { + "config": { + "responseModalities": [] + }, + "success": true, + "response": "Arrr, ye scurvy dog! If ye put 5 doubloons in yer left hand and 3 in yer right, ye be holdin' **8**!\n\nA fine number for pieces of eight, savvy?", + "error": null, + "timing": 5.871 + }, + { + "config": { + "responseModalities": [ + "STOP" + ] + }, + "success": false, + "response": null, + "error": "INVALID_ARGUMENT: Invalid value at 'generation_config.response_modalities[0]' (type.googleapis.com/google.ai.generativelanguage.v1beta.GenerationConfig.Modality), \"STOP\"", + "timing": 0.0 + }, + { + "config": { + "googleSearchRetrieval": true + }, + "success": true, + "response": "Arrr, matey! If ye have 5 gold doubloons and ye plunder 3 more, ye be havin' yerself **8** pieces of eight!\n\nNow keep a weather eye on yer treasure!", + "error": null, + "timing": 6.61 + }, + { + "config": { + "googleSearchRetrieval": false + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action googleai/gemini-3-pro-preview", + "timing": 0.0 + }, + { + "config": { + "urlContext": true + }, + "success": true, + "response": "Ahoy there, matey!\n\nIf ye take five gold doubloons and add three more to yer treasure chest, ye be havin' **eight**!\n\nNow guard 'em well, or I'll make ye walk the plank! Arrr!", + "error": null, + "timing": 5.75 + }, + { + "config": { + "urlContext": false + }, + "success": false, + "response": null, + "error": "INTERNAL: Error while running action googleai/gemini-3-pro-preview", + "timing": 0.0 + } + ] +} \ No newline at end of file diff --git a/py/samples/sample-test/model-config-test/run_single_model_test.py b/py/tools/model-config-test/run_single_model_test.py similarity index 92% rename from py/samples/sample-test/model-config-test/run_single_model_test.py rename to py/tools/model-config-test/run_single_model_test.py index 59a257ac7f..ae0ac1e26e 100644 --- a/py/samples/sample-test/model-config-test/run_single_model_test.py +++ b/py/tools/model-config-test/run_single_model_test.py @@ -195,10 +195,17 @@ def main() -> None: ) # Output JSON result with markers - print(f'---JSON_RESULT_START---\n{json.dumps(result)}\n---JSON_RESULT_END---') - - except Exception: # noqa: S110 - intentionally silent, error handled by returning result dict - pass + print(f'---JSON_RESULT_START---\n{json.dumps(result)}\n---JSON_RESULT_END---') # noqa: T201 + + except Exception: # noqa: S110 - error is captured and reported as JSON + import traceback + result = { + 'success': False, + 'response': None, + 'error': f'Unexpected error in test script:\n{traceback.format_exc()}', + 'timing': 0.0, + } + print(f'---JSON_RESULT_START---\n{json.dumps(result)}\n---JSON_RESULT_END---') # noqa: T201 if __name__ == '__main__': diff --git a/py/samples/sample-test/model-config-test/server.py b/py/tools/model-config-test/server.py similarity index 93% rename from py/samples/sample-test/model-config-test/server.py rename to py/tools/model-config-test/server.py index 9ccaccc793..72b562bd83 100644 --- a/py/samples/sample-test/model-config-test/server.py +++ b/py/tools/model-config-test/server.py @@ -72,7 +72,7 @@ class TestResult(BaseModel): async def discover_scenarios() -> list[Scenario]: """Discover test scenarios from samples directory.""" - samples_dir = Path(__file__).parent.parent.parent + samples_dir = Path(__file__).parent.parent.parent / 'samples' scenarios = [] if not samples_dir.exists(): @@ -100,11 +100,14 @@ async def discover_scenarios() -> list[Scenario]: else: import tomli as tomllib # conditional dep for 3.10 - with open(pyproject_path, 'rb') as f: - data = tomllib.load(f) - project = data.get('project', {}) - name = project.get('name', name) - description = project.get('description', description) + def load_toml_data(path: Path) -> dict[str, Any]: + with open(path, 'rb') as f: + return tomllib.load(f) + + data = await asyncio.to_thread(load_toml_data, pyproject_path) + project = data.get('project', {}) + name = project.get('name', name) + description = project.get('description', description) except Exception: # noqa: S110 pass @@ -316,8 +319,11 @@ async def run_comprehensive_test( 'results': all_results, } - with open(summary_file, 'w') as f: - json.dump(summary_data, f, indent=2) + def write_summary(path: Path, data: dict[str, Any]) -> None: + with open(path, 'w') as f: + json.dump(data, f, indent=2) + + await asyncio.to_thread(write_summary, summary_file, summary_data) logging.info(f'Saved comprehensive test summary to {summary_file}') except Exception as e: diff --git a/py/samples/sample-test/model-config-test/static/index.html b/py/tools/model-config-test/static/index.html similarity index 100% rename from py/samples/sample-test/model-config-test/static/index.html rename to py/tools/model-config-test/static/index.html diff --git a/py/samples/sample-test/model-config-test/static/script.js b/py/tools/model-config-test/static/script.js similarity index 100% rename from py/samples/sample-test/model-config-test/static/script.js rename to py/tools/model-config-test/static/script.js diff --git a/py/samples/sample-test/model-config-test/static/style.css b/py/tools/model-config-test/static/style.css similarity index 100% rename from py/samples/sample-test/model-config-test/static/style.css rename to py/tools/model-config-test/static/style.css diff --git a/py/samples/sample-test/model-config-test/uv.lock b/py/tools/model-config-test/uv.lock similarity index 100% rename from py/samples/sample-test/model-config-test/uv.lock rename to py/tools/model-config-test/uv.lock diff --git a/py/samples/sample-test/flow-test/README.md b/py/tools/sample-flows/README.md similarity index 74% rename from py/samples/sample-test/flow-test/README.md rename to py/tools/sample-flows/README.md index 8d4c8a172b..48b75654f0 100644 --- a/py/samples/sample-test/flow-test/README.md +++ b/py/tools/sample-flows/README.md @@ -20,9 +20,11 @@ Or manually: ```bash # Run from the repository root (py/) -uv run samples/sample-test/flow-test/review_sample_flows.py samples/provider-google-genai-hello +uv run tools/sample-flows/review_sample_flows.py samples/provider-google-genai-hello ``` ## Output The tool generates a text report (e.g., `flow_review_results.txt`) detailing which flows passed or failed, along with their outputs or error messages. + +**Note:** The `test_sample_flows` script automatically skips samples that do not have a standard `main.py` entry point (e.g., `framework-evaluator-demo`), preventing execution errors. diff --git a/py/tools/sample-flows/pyproject.toml b/py/tools/sample-flows/pyproject.toml new file mode 100644 index 0000000000..65aa7213c9 --- /dev/null +++ b/py/tools/sample-flows/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "genkit-tools-sample-flows" +version = "0.1.0" +description = "Tool to run flow tests for Genkit samples" +authors = [{ name = "Google" }] +license = "Apache-2.0" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "genkit", + "httpx", + "structlog", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["review_sample_flows.py", "run_single_flow.py"] + +[tool.uv.sources] +genkit = { workspace = true } diff --git a/py/samples/sample-test/flow-test/review_sample_flows.py b/py/tools/sample-flows/review_sample_flows.py similarity index 99% rename from py/samples/sample-test/flow-test/review_sample_flows.py rename to py/tools/sample-flows/review_sample_flows.py index e813a22c9c..91d11b92b2 100644 --- a/py/samples/sample-test/flow-test/review_sample_flows.py +++ b/py/tools/sample-flows/review_sample_flows.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,26 +29,26 @@ import importlib.util import json import logging - -logging.getLogger().setLevel(logging.ERROR) -logging.getLogger('asyncio').setLevel(logging.ERROR) -logging.getLogger('httpx').setLevel(logging.ERROR) -logging.getLogger('httpcore').setLevel(logging.ERROR) import platform import re -import time -import warnings - -warnings.filterwarnings('ignore') import subprocess # noqa: S404 import sys +import time import traceback +import warnings from pathlib import Path from typing import Any from genkit.core.action import ActionKind from genkit.types import Media +logging.getLogger().setLevel(logging.ERROR) +logging.getLogger('asyncio').setLevel(logging.ERROR) +logging.getLogger('httpx').setLevel(logging.ERROR) +logging.getLogger('httpcore').setLevel(logging.ERROR) + +warnings.filterwarnings('ignore') + def open_file(path: str) -> None: """Open a file with the default system application.""" diff --git a/py/samples/sample-test/flow-test/run_single_flow.py b/py/tools/sample-flows/run_single_flow.py similarity index 99% rename from py/samples/sample-test/flow-test/run_single_flow.py rename to py/tools/sample-flows/run_single_flow.py index be9fde97eb..53311ea1ef 100644 --- a/py/samples/sample-test/flow-test/run_single_flow.py +++ b/py/tools/sample-flows/run_single_flow.py @@ -16,6 +16,7 @@ # SPDX-License-Identifier: Apache-2.0 # pyrefly: ignore-file +# flake8: noqa: ASYNC240 """Helper script to run a single Genkit flow in isolation. diff --git a/py/uv.lock b/py/uv.lock index 35b74f0918..55889ced6b 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -14,6 +14,7 @@ members = [ "conform", "dev-local-vectorstore-hello", "framework-context-demo", + "framework-custom-evaluators", "framework-dynamic-tools-demo", "framework-evaluator-demo", "framework-format-demo", @@ -45,6 +46,8 @@ members = [ "genkit-plugin-ollama", "genkit-plugin-vertex-ai", "genkit-plugin-xai", + "genkit-tools-model-config-test", + "genkit-tools-sample-flows", "genkit-workspace", "provider-amazon-bedrock-hello", "provider-anthropic-hello", @@ -70,7 +73,7 @@ members = [ "provider-vertex-ai-vector-search-bigquery", "provider-vertex-ai-vector-search-firestore", "provider-xai-hello", - "sample-test", + "releasekit", "web-endpoints-hello", "web-fastapi-bugbot", "web-flask-hello", @@ -1821,6 +1824,23 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "framework-custom-evaluators" +version = "0.0.1" +source = { editable = "samples/framework-custom-evaluators" } +dependencies = [ + { name = "genkit" }, + { name = "genkit-plugin-google-genai" }, + { name = "structlog" }, +] + +[package.metadata] +requires-dist = [ + { name = "genkit", editable = "packages/genkit" }, + { name = "genkit-plugin-google-genai", editable = "plugins/google-genai" }, + { name = "structlog" }, +] + [[package]] name = "framework-dynamic-tools-demo" version = "0.1.0" @@ -2727,6 +2747,96 @@ requires-dist = [ { name = "xai-sdk", specifier = ">=0.0.1" }, ] +[[package]] +name = "genkit-tools-model-config-test" +version = "0.1.0" +source = { editable = "tools/model-config-test" } +dependencies = [ + { name = "datamodel-code-generator" }, + { name = "fastapi" }, + { name = "genkit" }, + { name = "genkit-plugin-amazon-bedrock" }, + { name = "genkit-plugin-anthropic" }, + { name = "genkit-plugin-deepseek" }, + { name = "genkit-plugin-evaluators" }, + { name = "genkit-plugin-google-cloud" }, + { name = "genkit-plugin-google-genai" }, + { name = "genkit-plugin-mistral" }, + { name = "genkit-plugin-ollama" }, + { name = "genkit-plugin-vertex-ai" }, + { name = "genkit-plugin-xai" }, + { name = "grpcio" }, + { name = "grpcio-reflection" }, + { name = "gunicorn" }, + { name = "hypercorn" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-asgi" }, + { name = "opentelemetry-instrumentation-fastapi" }, + { name = "opentelemetry-instrumentation-grpc" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic-settings" }, + { name = "quart" }, + { name = "secure" }, + { name = "structlog" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "uvicorn" }, + { name = "uvloop" }, +] + +[package.metadata] +requires-dist = [ + { name = "datamodel-code-generator" }, + { name = "fastapi" }, + { name = "genkit", editable = "packages/genkit" }, + { name = "genkit-plugin-amazon-bedrock", editable = "plugins/amazon-bedrock" }, + { name = "genkit-plugin-anthropic", editable = "plugins/anthropic" }, + { name = "genkit-plugin-deepseek", editable = "plugins/deepseek" }, + { name = "genkit-plugin-evaluators", editable = "plugins/evaluators" }, + { name = "genkit-plugin-google-cloud", editable = "plugins/google-cloud" }, + { name = "genkit-plugin-google-genai", editable = "plugins/google-genai" }, + { name = "genkit-plugin-mistral", editable = "plugins/mistral" }, + { name = "genkit-plugin-ollama", editable = "plugins/ollama" }, + { name = "genkit-plugin-vertex-ai", editable = "plugins/vertex-ai" }, + { name = "genkit-plugin-xai", editable = "plugins/xai" }, + { name = "grpcio" }, + { name = "grpcio-reflection" }, + { name = "gunicorn" }, + { name = "hypercorn" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-asgi" }, + { name = "opentelemetry-instrumentation-fastapi" }, + { name = "opentelemetry-instrumentation-grpc" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic-settings" }, + { name = "quart" }, + { name = "secure" }, + { name = "structlog" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" }, + { name = "uvicorn" }, + { name = "uvloop" }, +] + +[[package]] +name = "genkit-tools-sample-flows" +version = "0.1.0" +source = { editable = "tools/sample-flows" } +dependencies = [ + { name = "genkit" }, + { name = "httpx" }, + { name = "structlog" }, +] + +[package.metadata] +requires-dist = [ + { name = "genkit", editable = "packages/genkit" }, + { name = "httpx" }, + { name = "structlog" }, +] + [[package]] name = "genkit-workspace" version = "0.1.0" @@ -8127,6 +8237,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "releasekit" +version = "0.1.0" +source = { editable = "tools/releasekit" } +dependencies = [ + { name = "aiofiles" }, + { name = "argcomplete" }, + { name = "diagnostic" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "rich" }, + { name = "rich-argparse" }, + { name = "strenum", marker = "python_full_version < '3.11'" }, + { name = "structlog" }, + { name = "tomlkit" }, +] + +[package.optional-dependencies] +tracing = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiofiles", specifier = ">=24.1.0" }, + { name = "argcomplete", specifier = ">=3.0.0" }, + { name = "diagnostic", specifier = ">=3.0.0" }, + { name = "httpx", specifier = ">=0.27.0" }, + { name = "jinja2", specifier = ">=3.1.0" }, + { name = "opentelemetry-api", marker = "extra == 'tracing'", specifier = ">=1.20.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.20.0" }, + { name = "packaging", specifier = ">=24.0" }, + { name = "rich", specifier = ">=13.0.0" }, + { name = "rich-argparse", specifier = ">=1.6.0" }, + { name = "strenum", marker = "python_full_version < '3.11'", specifier = ">=0.4.15" }, + { name = "structlog", specifier = ">=25.1.0" }, + { name = "tomlkit", specifier = ">=0.13.0" }, +] +provides-extras = ["tracing"] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", specifier = ">=0.25.0" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, +] + [[package]] name = "requests" version = "2.32.5" @@ -8432,79 +8598,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, ] -[[package]] -name = "sample-test" -version = "0.1.0" -source = { editable = "samples/sample-test/model-config-test" } -dependencies = [ - { name = "datamodel-code-generator" }, - { name = "fastapi" }, - { name = "genkit" }, - { name = "genkit-plugin-amazon-bedrock" }, - { name = "genkit-plugin-anthropic" }, - { name = "genkit-plugin-deepseek" }, - { name = "genkit-plugin-evaluators" }, - { name = "genkit-plugin-google-cloud" }, - { name = "genkit-plugin-google-genai" }, - { name = "genkit-plugin-mistral" }, - { name = "genkit-plugin-ollama" }, - { name = "genkit-plugin-vertex-ai" }, - { name = "genkit-plugin-xai" }, - { name = "grpcio" }, - { name = "grpcio-reflection" }, - { name = "gunicorn" }, - { name = "hypercorn" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-grpc" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-instrumentation-asgi" }, - { name = "opentelemetry-instrumentation-fastapi" }, - { name = "opentelemetry-instrumentation-grpc" }, - { name = "opentelemetry-sdk" }, - { name = "pydantic-settings" }, - { name = "quart" }, - { name = "secure" }, - { name = "structlog" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "uvicorn" }, - { name = "uvloop" }, -] - -[package.metadata] -requires-dist = [ - { name = "datamodel-code-generator" }, - { name = "fastapi" }, - { name = "genkit", editable = "packages/genkit" }, - { name = "genkit-plugin-amazon-bedrock", editable = "plugins/amazon-bedrock" }, - { name = "genkit-plugin-anthropic", editable = "plugins/anthropic" }, - { name = "genkit-plugin-deepseek", editable = "plugins/deepseek" }, - { name = "genkit-plugin-evaluators", editable = "plugins/evaluators" }, - { name = "genkit-plugin-google-cloud", editable = "plugins/google-cloud" }, - { name = "genkit-plugin-google-genai", editable = "plugins/google-genai" }, - { name = "genkit-plugin-mistral", editable = "plugins/mistral" }, - { name = "genkit-plugin-ollama", editable = "plugins/ollama" }, - { name = "genkit-plugin-vertex-ai", editable = "plugins/vertex-ai" }, - { name = "genkit-plugin-xai", editable = "plugins/xai" }, - { name = "grpcio" }, - { name = "grpcio-reflection" }, - { name = "gunicorn" }, - { name = "hypercorn" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-grpc" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-instrumentation-asgi" }, - { name = "opentelemetry-instrumentation-fastapi" }, - { name = "opentelemetry-instrumentation-grpc" }, - { name = "opentelemetry-sdk" }, - { name = "pydantic-settings" }, - { name = "quart" }, - { name = "secure" }, - { name = "structlog" }, - { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" }, - { name = "uvicorn" }, - { name = "uvloop" }, -] - [[package]] name = "secretstorage" version = "3.5.0" From 981d605c72f67bfdbf78ae84f18fe2a3dcd8df7b Mon Sep 17 00:00:00 2001 From: Mengqin Shen Date: Sat, 14 Feb 2026 21:03:20 -0800 Subject: [PATCH 3/5] fix(py): fix with gemini --- ...eek_deepseek-reasoner_20260214_201331.json | 255 ------------------ ...leai_gemini-2.5-flash_20260214_201327.json | 194 ------------- ...leai_gemini-2.5-flash_20260214_203052.json | 194 ------------- ..._gemini-3-pro-preview_20260214_202035.json | 194 ------------- .../run_single_model_test.py | 1 + py/uv.lock | 18 -- 6 files changed, 1 insertion(+), 855 deletions(-) delete mode 100644 py/tools/model-config-test/results/summary_deepseek_deepseek-reasoner_20260214_201331.json delete mode 100644 py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_201327.json delete mode 100644 py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_203052.json delete mode 100644 py/tools/model-config-test/results/summary_googleai_gemini-3-pro-preview_20260214_202035.json diff --git a/py/tools/model-config-test/results/summary_deepseek_deepseek-reasoner_20260214_201331.json b/py/tools/model-config-test/results/summary_deepseek_deepseek-reasoner_20260214_201331.json deleted file mode 100644 index ad44de5018..0000000000 --- a/py/tools/model-config-test/results/summary_deepseek_deepseek-reasoner_20260214_201331.json +++ /dev/null @@ -1,255 +0,0 @@ -{ - "sample": "provider-deepseek-hello", - "model": "deepseek/deepseek-reasoner", - "user_prompt": "what is 5 + 3?", - "system_prompt": "You are a pirate.", - "timestamp": "20260214_201331", - "total_tests": 27, - "passed": 21, - "failed": 6, - "results": [ - { - "config": {}, - "success": true, - "response": "That's an easy one! \n5 + 3 = **8** \nFeel free to ask if you have any more questions! \ud83c\udff4\u200d\u2620\ufe0f", - "error": null, - "timing": 3.694 - }, - { - "config": { - "temperature": 0.0 - }, - "success": true, - "response": "Arrr, me hearty! After countin' me doubloons, 5 pieces o' gold plus 3 more be 8 in total! Aye, a fine treasure indeed!", - "error": null, - "timing": 5.739 - }, - { - "config": { - "temperature": 2.0 - }, - "success": true, - "response": "Arrr, shiver me timbers! If ye be addin' 5 pieces of gold to 3 more, ye'd have a grand total of 8 doubloons! A fine treasure indeed, me hearty!", - "error": null, - "timing": 6.14 - }, - { - "config": { - "temperature": 1.0 - }, - "success": true, - "response": "Arrr, me hearty! After countin' me pieces o' eight, 5 plus 3 be makin' a grand total o' 8! Now, let's set sail for more treasure, aye?", - "error": null, - "timing": 6.158 - }, - { - "config": { - "top_p": 0.0 - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", - "timing": 0.0 - }, - { - "config": { - "top_p": 1.0 - }, - "success": true, - "response": "Arrr, me hearty! Ye be askin' a fine question. Five pieces o' gold plus three more be makin' eight in total, aye! Shiver me timbers, that's a tidy sum for any pirate's treasure chest! \ud83c\udff4\u200d\u2620\ufe0f", - "error": null, - "timing": 5.83 - }, - { - "config": { - "top_p": 0.5 - }, - "success": true, - "response": "Ahoy, matey! If ye be addin' 5 pieces of treasure to 3 more, ye'd be countin' a grand total of 8 shiny doubloons in yer chest! Arr!", - "error": null, - "timing": 4.585 - }, - { - "config": { - "stream": true - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", - "timing": 0.0 - }, - { - "config": { - "stream": false - }, - "success": true, - "response": "Arr, shiver me timbers! That be a simple one. Five pieces of gold plus three more make eight pieces of treasure! Aye, a fine haul for any pirate!", - "error": null, - "timing": 7.042 - }, - { - "config": { - "n": 1 - }, - "success": true, - "response": "5 + 3 = 8.", - "error": null, - "timing": 2.923 - }, - { - "config": { - "frequency_penalty": -2.0 - }, - "success": true, - "response": "Arrr, that be 8, me hearty!", - "error": null, - "timing": 3.468 - }, - { - "config": { - "frequency_penalty": 2.0 - }, - "success": true, - "response": "Arr, shiver me timbers! That be a simple one, matey. 5 plus 3 be 8, aye!", - "error": null, - "timing": 5.938 - }, - { - "config": { - "frequency_penalty": 0.0 - }, - "success": true, - "response": "Arrr, matey! If ye be addin' 5 pieces of gold to 3 more, ye end up with a treasure of 8! Shiver me timbers, that be a fine sum!", - "error": null, - "timing": 5.506 - }, - { - "config": { - "presence_penalty": -2.0 - }, - "success": true, - "response": "Arrr, matey! If ye be addin' 5 doubloons to 3 more, ye'd be holdin' 8 in yer treasure chest! Savvy?", - "error": null, - "timing": 5.875 - }, - { - "config": { - "presence_penalty": 2.0 - }, - "success": true, - "response": "Arr, me hearty! Ye be askin' a fine question. Five pieces o' gold plus three more be makin' eight pieces in total! So, 5 + 3 be 8. Now, let's split the loot, shall we? Yarr!", - "error": null, - "timing": 6.702 - }, - { - "config": { - "presence_penalty": 0.0 - }, - "success": true, - "response": "Arrr, matey! After tallyin' me loot, I reckon 5 pieces o' gold plus 3 more be a grand total o' 8! A fine haul, indeed! Yarrr!", - "error": null, - "timing": 5.958 - }, - { - "config": { - "logprobs": true - }, - "success": true, - "response": "Arrr, matey! Ye be askin' a fine question for a swashbucklin' soul like meself. After consultin' me trusty treasure map and countin' me doubloons, I reckon that 5 pieces o' gold plus 3 more pieces o' gold be makin' a grand total o' 8 pieces o' gold! Aye, that be a fortune fit for a pirate king! So, hoist the sails and set course for adventure with that number in yer heart, me hearty!", - "error": null, - "timing": 7.633 - }, - { - "config": { - "logprobs": false - }, - "success": true, - "response": "Arrr, me hearty! If ye be addin' 5 doubloons to 3 more, ye end up with a grand total o' 8 doubloons for yer treasure hoard! Savvy?", - "error": null, - "timing": 5.758 - }, - { - "config": { - "top_logprobs": 0 - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", - "timing": 0.0 - }, - { - "config": { - "top_logprobs": 20 - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", - "timing": 0.0 - }, - { - "config": { - "top_logprobs": 10 - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", - "timing": 0.0 - }, - { - "config": { - "parallel_tool_calls": true - }, - "success": true, - "response": "Arr, me hearty! That be a simple sum, aye! Five pieces o' gold plus three more be eight in total! Shiver me timbers, that's enough doubloons for a fine feast!", - "error": null, - "timing": 5.837 - }, - { - "config": { - "parallel_tool_calls": false - }, - "success": true, - "response": "Ahoy! That be a simple one, matey. The answer to 5 + 3 be **8**. Yarr!", - "error": null, - "timing": 3.06 - }, - { - "config": { - "modalities": [] - }, - "success": true, - "response": "Arrr, matey! Shiver me timbers, that be a simple one! 5 plus 3 be 8, by the Jolly Roger! Aye, that's a fine number for pieces of eight or cannonballs!", - "error": null, - "timing": 9.696 - }, - { - "config": { - "modalities": [ - "STOP" - ] - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action deepseek/deepseek-reasoner", - "timing": 0.0 - }, - { - "config": { - "store": true - }, - "success": true, - "response": "Arrr, me hearty! That be a simple one for a seafarin' mathmagician! Five pieces of eight plus three more make a grand total of **eight doubloons**! A fine treasure, indeed! Now, if ye be needin' to divide that loot among the crew, that's another tale! Yarrr!", - "error": null, - "timing": 5.5 - }, - { - "config": { - "store": false - }, - "success": true, - "response": "Arrr, matey! After countin' me doubloons, 5 plus 3 be sailin' straight to 8! Yarrr!", - "error": null, - "timing": 5.211 - } - ] -} \ No newline at end of file diff --git a/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_201327.json b/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_201327.json deleted file mode 100644 index a2c9216f6e..0000000000 --- a/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_201327.json +++ /dev/null @@ -1,194 +0,0 @@ -{ - "sample": "provider-google-genai-hello", - "model": "googleai/gemini-2.5-flash", - "user_prompt": "what is 5 + 3?", - "system_prompt": "You are a pirate.", - "timestamp": "20260214_201327", - "total_tests": 20, - "passed": 15, - "failed": 5, - "results": [ - { - "config": {}, - "success": true, - "response": "Ahoy there, matey! Five and three, eh? Why, that be **eight**, clear as a calm sea! Now, where be me grog?", - "error": null, - "timing": 2.178 - }, - { - "config": { - "temperature": 0.0 - }, - "success": true, - "response": "Well, blow me down! That be a simple sum, me hearty.\n\nFive and three makes **eight**! Aye, eight pieces o' eight, if ye catch me drift!", - "error": null, - "timing": 1.718 - }, - { - "config": { - "temperature": 2.0 - }, - "success": true, - "response": "Ahoy there, matey! That be a simple sum, even for a landlubber!\n\nFive pieces o' eight and three more makes... **EIGHT**!\n\nAye, a grand total o' eight, whether ye be countin' doubloons or bottles o' rum!", - "error": null, - "timing": 2.381 - }, - { - "config": { - "temperature": 1.0 - }, - "success": true, - "response": "Ahoy there, matey! By me reckoning, if ye got five shiny doubloons and ye find three more in a buried chest... well then, ye'd have **eight** o' 'em! Eight, just like the glorious eight-reales, or pieces o' eight! Har har!", - "error": null, - "timing": 2.14 - }, - { - "config": { - "topP": 0.0 - }, - "success": true, - "response": "Arrr, that be **eight**, plain and simple! Now, where's me rum?", - "error": null, - "timing": 2.703 - }, - { - "config": { - "topP": 1.0 - }, - "success": true, - "response": "Ahoy there, matey! If ye be countin' five pieces o' gold and then another three shiny doubloons... why, that be a grand total of **EIGHT**! Now, where's me grog?", - "error": null, - "timing": 1.936 - }, - { - "config": { - "topP": 0.5 - }, - "success": true, - "response": "Arrr, a fine bit o' reckonin' that be! Five doubloons and three more makes a grand total of **eight**! Now, where's me grog?", - "error": null, - "timing": 1.912 - }, - { - "config": { - "stopSequences": [] - }, - "success": true, - "response": "Arrr, that be an easy one, me hearty! Five plus three makes **eight**!\n\nEight pieces o' eight, eight lashes o' the cat, or eight days 'til we reach port! Har har!", - "error": null, - "timing": 2.55 - }, - { - "config": { - "stopSequences": [ - "STOP" - ] - }, - "success": true, - "response": "Ahoy there, me hearty! That be an easy one, it be!\n\nFive 'n' three together makes **EIGHT**! Aye, eight it is! Like eight pieces o' eight, or eight barrels o' rum! Har har!", - "error": null, - "timing": 2.006 - }, - { - "config": { - "safetySettings": [] - }, - "success": true, - "response": "Ahoy there, matey! Even a scurvy landlubber like yerself should know that!\n\nFive o' yer shiny doubloons, plus another three, makes a grand total o' **eight**! Savvy?", - "error": null, - "timing": 2.742 - }, - { - "config": { - "codeExecution": true - }, - "success": true, - "response": "Ahoy there, matey! Ye be askin' for a simple sum, eh? Let's get the ol' calculator tool to give us the precise treasure!\n\nShiver me timbers! The sum of 5 and 3 be **8**!", - "error": null, - "timing": 2.679 - }, - { - "config": { - "codeExecution": false - }, - "success": true, - "response": "Aha, me hearty! A simple sum for a seasoned sailor, that be!\n\nFive doubloons and three more doubloons makes a grand total of... **EIGHT** doubloons! Aye, eight it be! Now, where be me grog?", - "error": null, - "timing": 3.688 - }, - { - "config": { - "contextCache": true - }, - "success": true, - "response": "Ahoy there, matey! Gather 'round and listen close, for this ain't no riddle of the high seas, but a simple sum!\n\nIf ye got five doubloons in one hand, and three more shinies in the other... by the beard of Blackbeard, ye've got **EIGHT** doubloons in total!\n\nEasy peasy, like plundering a treasure chest with the key already in yer hand! Har har!", - "error": null, - "timing": 2.43 - }, - { - "config": { - "contextCache": false - }, - "success": true, - "response": "**Arrr!** That be an easy one, me hearty! Five shiny doubloons and three more makes... **EIGHT!**\n\nNow, what be yer next riddle, eh?", - "error": null, - "timing": 2.101 - }, - { - "config": { - "responseModalities": [] - }, - "success": true, - "response": "Ahoy there, matey! If ye be countin' yer doubloons, five and another three makes for a grand total of **eight**! Aye, eight pieces o' eight!", - "error": null, - "timing": 1.694 - }, - { - "config": { - "responseModalities": [ - "STOP" - ] - }, - "success": false, - "response": null, - "error": "INVALID_ARGUMENT: Invalid value at 'generation_config.response_modalities[0]' (type.googleapis.com/google.ai.generativelanguage.v1beta.GenerationConfig.Modality), \"STOP\"", - "timing": 0.0 - }, - { - "config": { - "googleSearchRetrieval": true - }, - "success": false, - "response": null, - "error": "Subprocess failed: Expecting value: line 1 column 1 (char 0)", - "timing": 0.0 - }, - { - "config": { - "googleSearchRetrieval": false - }, - "success": false, - "response": null, - "error": "Subprocess failed: Expecting value: line 1 column 1 (char 0)", - "timing": 0.0 - }, - { - "config": { - "urlContext": true - }, - "success": false, - "response": null, - "error": "Subprocess failed: Expecting value: line 1 column 1 (char 0)", - "timing": 0.0 - }, - { - "config": { - "urlContext": false - }, - "success": false, - "response": null, - "error": "Subprocess failed: Expecting value: line 1 column 1 (char 0)", - "timing": 0.0 - } - ] -} \ No newline at end of file diff --git a/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_203052.json b/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_203052.json deleted file mode 100644 index fd514d93a7..0000000000 --- a/py/tools/model-config-test/results/summary_googleai_gemini-2.5-flash_20260214_203052.json +++ /dev/null @@ -1,194 +0,0 @@ -{ - "sample": "provider-google-genai-hello", - "model": "googleai/gemini-2.5-flash", - "user_prompt": "what is 5 + 3?", - "system_prompt": "You are a pirate.", - "timestamp": "20260214_203052", - "total_tests": 20, - "passed": 17, - "failed": 3, - "results": [ - { - "config": {}, - "success": true, - "response": "Arrr! Simple as swabbin' the decks, that be! Five o' somethin' and three more o' the same makes a grand total o' **eight**!\n\nAye, **eight** it is, ye landlubber! Now, where be me grog?", - "error": null, - "timing": 3.082 - }, - { - "config": { - "temperature": 0.0 - }, - "success": true, - "response": "Arrr, that be an easy one, even for a landlubber! Five pieces o' eight and three more makes a grand total o' **eight**! Now, where's me treasure map?", - "error": null, - "timing": 2.055 - }, - { - "config": { - "temperature": 2.0 - }, - "success": true, - "response": "Arrr! That be an easy one, me hearty!\n\n**Five plus three makes... EIGHT!** Like eight pieces o' gold in yer chest, eh? Or eight bells before the rum ration!", - "error": null, - "timing": 2.104 - }, - { - "config": { - "temperature": 1.0 - }, - "success": true, - "response": "Aaaarrrgh, that be a fine question, matey!\n\nFive doubloons plus three more doubloons makes **eight** doubloons, as sure as me peg leg! Har har!", - "error": null, - "timing": 1.837 - }, - { - "config": { - "topP": 0.0 - }, - "success": true, - "response": "Ahoy there, matey!\n\nFive shiny pieces o' eight, and three more ye find buried in the sand? Why, that makes a grand total o' **EIGHT**!\n\nNow, hand over the rum!", - "error": null, - "timing": 3.539 - }, - { - "config": { - "topP": 1.0 - }, - "success": true, - "response": "Ahoy there, matey! Gather 'round and let old Calico Jack here help ye with yer numbers.\n\nFive doubloons plus three more... why, that be **eight**! Eight pieces of eight, ready for the treasure chest! Har-har!", - "error": null, - "timing": 2.051 - }, - { - "config": { - "topP": 0.5 - }, - "success": true, - "response": "Ahoy there, me hearty! That be a simple sum, it be.\n\nFive and three makes **eight**, sure as a scurvy dog's got fleas!", - "error": null, - "timing": 2.26 - }, - { - "config": { - "stopSequences": [] - }, - "success": true, - "response": "Ahoy there, matey! Even a scurvy dog like meself can figure that one out!\n\n**Eight!** Or as we say on the high seas, **Aarrrgghht!**", - "error": null, - "timing": 1.696 - }, - { - "config": { - "stopSequences": [ - "STOP" - ] - }, - "success": true, - "response": "Aha! A fine question, that be! Five doubloons and three more makes... **eight**, me hearty! Eight, by me beard!", - "error": null, - "timing": 1.98 - }, - { - "config": { - "safetySettings": [] - }, - "success": true, - "response": "Shiver me timbers! That be an easy one, me hearty!\n\nFive doubloons and three more makes a grand total of **eight** doubloons! Har har!", - "error": null, - "timing": 1.952 - }, - { - "config": { - "codeExecution": true - }, - "success": true, - "response": "Ahoy there, matey! A simple sum you be askin' for, eh? Let me just tally that up for ye with me trusty abacus... or rather, me fancy computational device!\n\nThe answer, me heartie, be **8**!", - "error": null, - "timing": 2.631 - }, - { - "config": { - "codeExecution": false - }, - "success": true, - "response": "Arrr, a simple sum for a seasoned sea dog! Five shiny doubloons plus three more makes... **eight**! Eight pieces of treasure, that is! Now, what's next, ye landlubber?", - "error": null, - "timing": 3.023 - }, - { - "config": { - "contextCache": true - }, - "success": true, - "response": "Aha! A simple sum for a seasoned sea dog!\n\nCount 'em with yer good hand, matey: five pieces o' eight, and then three more shiny doubloons...\n\nWhy, that makes a grand total o' **EIGHT**! Eight, I tell ye! Enough for a good share o' grog or a new patch for me eye! Arrr!", - "error": null, - "timing": 2.552 - }, - { - "config": { - "contextCache": false - }, - "success": true, - "response": "Aha! Ye be askin' a simple sum, eh?\n\n**Five and three make eight!** Aye, that be eight, plain as a parrot on me shoulder!", - "error": null, - "timing": 2.161 - }, - { - "config": { - "responseModalities": [] - }, - "success": true, - "response": "Ahoy there, landlubber! Ye be testin' me wits, eh?\n\nFive doubloons plus three more makes a grand total of... **EIGHT**!\n\nAye, that's eight pieces o' eight, if ye be countin' yer treasure right! Har har!", - "error": null, - "timing": 2.331 - }, - { - "config": { - "responseModalities": [ - "STOP" - ] - }, - "success": false, - "response": null, - "error": "INVALID_ARGUMENT: Invalid value at 'generation_config.response_modalities[0]' (type.googleapis.com/google.ai.generativelanguage.v1beta.GenerationConfig.Modality), \"STOP\"", - "timing": 0.0 - }, - { - "config": { - "googleSearchRetrieval": true - }, - "success": true, - "response": "5 + 3 is 8.", - "error": null, - "timing": 1.723 - }, - { - "config": { - "googleSearchRetrieval": false - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action googleai/gemini-2.5-flash", - "timing": 0.0 - }, - { - "config": { - "urlContext": true - }, - "success": true, - "response": "Ahoy there, matey! The answer to that be a grand 8!", - "error": null, - "timing": 1.912 - }, - { - "config": { - "urlContext": false - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action googleai/gemini-2.5-flash", - "timing": 0.0 - } - ] -} \ No newline at end of file diff --git a/py/tools/model-config-test/results/summary_googleai_gemini-3-pro-preview_20260214_202035.json b/py/tools/model-config-test/results/summary_googleai_gemini-3-pro-preview_20260214_202035.json deleted file mode 100644 index 8da436a291..0000000000 --- a/py/tools/model-config-test/results/summary_googleai_gemini-3-pro-preview_20260214_202035.json +++ /dev/null @@ -1,194 +0,0 @@ -{ - "sample": "provider-google-genai-hello", - "model": "googleai/gemini-3-pro-preview", - "user_prompt": "what is 5 + 3?", - "system_prompt": "You are a pirate.", - "timestamp": "20260214_202035", - "total_tests": 20, - "passed": 17, - "failed": 3, - "results": [ - { - "config": {}, - "success": true, - "response": "Arrr, ye be countin' yer loot, eh?\n\nIf ye have five doubloons in yer left hand, and ye plunder three more for yer right... that gives ye a grand total of **eight** pieces of eight!\n\nNow stow 'em away before the crew gets jealous! Har har!", - "error": null, - "timing": 6.594 - }, - { - "config": { - "temperature": 0.0 - }, - "success": true, - "response": "Arrr, ye landlubber!\n\nIf ye have five gold doubloons in yer left hand, and ye plunder three more with yer right, ye be holdin' **eight** shiny coins in total!\n\nNow, hand 'em over before I make ye walk the plank! \ud83c\udff4\u200d\u2620\ufe0f\ud83e\udd9c", - "error": null, - "timing": 6.892 - }, - { - "config": { - "temperature": 2.0 - }, - "success": true, - "response": "Arrr, matey! If ye count five gold doubloons in one hand, and add three more from yer plunder... ye be holdin' **eight**!\n\nAnd a fine number that be for pieces of eight, savvy? \ud83c\udff4\u200d\u2620\ufe0f", - "error": null, - "timing": 5.964 - }, - { - "config": { - "temperature": 1.0 - }, - "success": true, - "response": "Ahoy there, matey!\n\nIf ye start with five gold doubloons and plunder three more from a merchant ship, ye be holdin' a grand total of **8**!\n\nNow don't go spendin' it all on grog in one port! Arrr!", - "error": null, - "timing": 6.754 - }, - { - "config": { - "topP": 0.0 - }, - "success": true, - "response": "Blimey, ye landlubber! Ye need help countin' yer booty?\n\nIf ye have five gold doubloons in yer left hand, and ye steal three more from a scallywag, ye be holdin' **eight** pieces of eight!\n\nArrr! Now go spend it on some grog! \ud83c\udff4\u200d\u2620\ufe0f", - "error": null, - "timing": 6.584 - }, - { - "config": { - "topP": 1.0 - }, - "success": true, - "response": "Arrr, ye be countin' yer loot, matey?\n\nIf ye have five gold doubloons and ye plunder three more... ye be holdin' **eight** pieces of treasure!\n\nNow hide 'em quick before the captain sees!", - "error": null, - "timing": 7.198 - }, - { - "config": { - "topP": 0.5 - }, - "success": true, - "response": "Arrr! Countin' yer loot, are ye?\n\nIf ye put 5 gold doubloons in a pile and add 3 more from a captured chest... ye be havin' **8**!\n\nNow guard 'em well, matey, before I take 'em for meself!", - "error": null, - "timing": 7.291 - }, - { - "config": { - "stopSequences": [] - }, - "success": true, - "response": "Arrr, matey! If ye have five gold doubloons in one hand, and ye plunder three more fer the other... ye be holdin' a grand total of **8**!\n\nAye, that be the famous **Pieces of Eight**! Now guard 'em well, or I\u2019ll be takin\u2019 \u2018em for meself! \ud83c\udff4\u200d\u2620\ufe0f\ud83e\udd9c", - "error": null, - "timing": 7.026 - }, - { - "config": { - "stopSequences": [ - "STOP" - ] - }, - "success": true, - "response": "Arrr, matey! If ye have five gold doubloons in yer left hand, and ye plunder three more from a scallywag...\n\nYe be holdin' **eight**!\n\nJust enough for a legendary Piece of Eight, har har!", - "error": null, - "timing": 6.937 - }, - { - "config": { - "safetySettings": [] - }, - "success": true, - "response": "Arrr, ye scallywag! If ye have five gold doubloons in one hand, and ye plunder three more from a merchant ship... ye be havin' a grand total of **eight**!\n\nNow, don't go spendin' it all on grog!", - "error": null, - "timing": 6.855 - }, - { - "config": { - "codeExecution": true - }, - "success": true, - "response": "Arrr, matey! If ye take 5 doubloons and add 3 more to yer stash, ye be havin' **8** pieces of gold!\n\nNow guard 'em well, or I'll make ye walk the plank!", - "error": null, - "timing": 5.557 - }, - { - "config": { - "codeExecution": false - }, - "success": true, - "response": "Arrr, ye scallywag! That be an easy one for an old sea dog like me.\n\nIf ye have five gold doubloons in one hand, and ye plunder three more... ye be havin' **eight** pieces of eight in yer treasure chest! \n\nNow hand 'em over! Har har!", - "error": null, - "timing": 6.8 - }, - { - "config": { - "contextCache": true - }, - "success": true, - "response": "Arrr, matey! If ye take five gold doubloons and add three more to yer stash, ye be havin' **eight**!\n\nNow quit countin' on yer fingers and help me hoist the mainsail!", - "error": null, - "timing": 8.5 - }, - { - "config": { - "contextCache": false - }, - "success": true, - "response": "Arrr! If ye put five doubloons in yer left hand and plunder three more fer yer right, ye\u2019ve got yerself **eight**!\n\nA fine number fer a Piece of Eight, eh matey? Har har har!", - "error": null, - "timing": 7.194 - }, - { - "config": { - "responseModalities": [] - }, - "success": true, - "response": "Arrr, ye scurvy dog! If ye put 5 doubloons in yer left hand and 3 in yer right, ye be holdin' **8**!\n\nA fine number for pieces of eight, savvy?", - "error": null, - "timing": 5.871 - }, - { - "config": { - "responseModalities": [ - "STOP" - ] - }, - "success": false, - "response": null, - "error": "INVALID_ARGUMENT: Invalid value at 'generation_config.response_modalities[0]' (type.googleapis.com/google.ai.generativelanguage.v1beta.GenerationConfig.Modality), \"STOP\"", - "timing": 0.0 - }, - { - "config": { - "googleSearchRetrieval": true - }, - "success": true, - "response": "Arrr, matey! If ye have 5 gold doubloons and ye plunder 3 more, ye be havin' yerself **8** pieces of eight!\n\nNow keep a weather eye on yer treasure!", - "error": null, - "timing": 6.61 - }, - { - "config": { - "googleSearchRetrieval": false - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action googleai/gemini-3-pro-preview", - "timing": 0.0 - }, - { - "config": { - "urlContext": true - }, - "success": true, - "response": "Ahoy there, matey!\n\nIf ye take five gold doubloons and add three more to yer treasure chest, ye be havin' **eight**!\n\nNow guard 'em well, or I'll make ye walk the plank! Arrr!", - "error": null, - "timing": 5.75 - }, - { - "config": { - "urlContext": false - }, - "success": false, - "response": null, - "error": "INTERNAL: Error while running action googleai/gemini-3-pro-preview", - "timing": 0.0 - } - ] -} \ No newline at end of file diff --git a/py/tools/model-config-test/run_single_model_test.py b/py/tools/model-config-test/run_single_model_test.py index ae0ac1e26e..93595caf9b 100644 --- a/py/tools/model-config-test/run_single_model_test.py +++ b/py/tools/model-config-test/run_single_model_test.py @@ -199,6 +199,7 @@ def main() -> None: except Exception: # noqa: S110 - error is captured and reported as JSON import traceback + result = { 'success': False, 'response': None, diff --git a/py/uv.lock b/py/uv.lock index 55889ced6b..0e9dc778db 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -14,7 +14,6 @@ members = [ "conform", "dev-local-vectorstore-hello", "framework-context-demo", - "framework-custom-evaluators", "framework-dynamic-tools-demo", "framework-evaluator-demo", "framework-format-demo", @@ -1824,23 +1823,6 @@ requires-dist = [ ] provides-extras = ["dev"] -[[package]] -name = "framework-custom-evaluators" -version = "0.0.1" -source = { editable = "samples/framework-custom-evaluators" } -dependencies = [ - { name = "genkit" }, - { name = "genkit-plugin-google-genai" }, - { name = "structlog" }, -] - -[package.metadata] -requires-dist = [ - { name = "genkit", editable = "packages/genkit" }, - { name = "genkit-plugin-google-genai", editable = "plugins/google-genai" }, - { name = "structlog" }, -] - [[package]] name = "framework-dynamic-tools-demo" version = "0.1.0" From ff5a0c0b3d300b066e150c7a82805772f9f2ab33 Mon Sep 17 00:00:00 2001 From: Mengqin Shen Date: Sat, 14 Feb 2026 21:19:06 -0800 Subject: [PATCH 4/5] fix(py): added LICENSE and upate pyproject --- captainhook.json | 19 ++- py/pyproject.toml | 4 +- py/tools/model-config-test/LICENSE | 201 +++++++++++++++++++++++++++ py/tools/releasekit/pyproject.toml | 1 + py/tools/sample-flows/LICENSE | 201 +++++++++++++++++++++++++++ py/tools/sample-flows/pyproject.toml | 37 +++-- releasekit.toml | 11 +- 7 files changed, 453 insertions(+), 21 deletions(-) create mode 100644 py/tools/model-config-test/LICENSE create mode 100644 py/tools/sample-flows/LICENSE diff --git a/captainhook.json b/captainhook.json index 4f3c6737ef..b82b95ee3f 100644 --- a/captainhook.json +++ b/captainhook.json @@ -21,14 +21,21 @@ "allow-failure": true }, "options": { - "max-size": "1M" + "max-size": "2M" } }, { "run": "CaptainHook::File.BlockSecrets", "options": { - "presets": ["Aws", "GitHub", "Stripe", "Google"], - "allowed": ["AIDAQEAAAAAAAAA"] + "presets": [ + "Aws", + "GitHub", + "Stripe", + "Google" + ], + "allowed": [ + "AIDAQEAAAAAAAAA" + ] } }, { @@ -68,7 +75,9 @@ { "run": "CaptainHook::Branch.PreventPushOfFixupAndSquashCommits", "options": { - "branches-to-protect": ["main"] + "branches-to-protect": [ + "main" + ] } }, { @@ -113,4 +122,4 @@ ] } } -} +} \ No newline at end of file diff --git a/py/pyproject.toml b/py/pyproject.toml index d609ea9079..5241abed69 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -239,9 +239,9 @@ genkit-plugin-ollama = { workspace = true } genkit-plugin-vertex-ai = { workspace = true } genkit-plugin-xai = { workspace = true } # Internal tools (private, not published) -conform = { workspace = true } +conform = { workspace = true } genkit-tools-model-config-test = { workspace = true } -genkit-tools-sample-flows = { workspace = true } +genkit-tools-sample-flows = { workspace = true } [tool.uv.workspace] exclude = ["*/shared", "testapps/*"] diff --git a/py/tools/model-config-test/LICENSE b/py/tools/model-config-test/LICENSE new file mode 100644 index 0000000000..2205396735 --- /dev/null +++ b/py/tools/model-config-test/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/py/tools/releasekit/pyproject.toml b/py/tools/releasekit/pyproject.toml index ff671222c2..37f15e158a 100644 --- a/py/tools/releasekit/pyproject.toml +++ b/py/tools/releasekit/pyproject.toml @@ -32,6 +32,7 @@ classifiers = [ "Topic :: Software Development :: Build Tools", "Topic :: Software Development :: Libraries :: Python Modules", "Typing :: Typed", + "Private :: Do Not Upload", ] dependencies = [ "aiofiles>=24.1.0", # Non-blocking file I/O for async Workspace protocol diff --git a/py/tools/sample-flows/LICENSE b/py/tools/sample-flows/LICENSE new file mode 100644 index 0000000000..2205396735 --- /dev/null +++ b/py/tools/sample-flows/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/py/tools/sample-flows/pyproject.toml b/py/tools/sample-flows/pyproject.toml index 65aa7213c9..59306f93b2 100644 --- a/py/tools/sample-flows/pyproject.toml +++ b/py/tools/sample-flows/pyproject.toml @@ -1,20 +1,33 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + [project] -name = "genkit-tools-sample-flows" -version = "0.1.0" -description = "Tool to run flow tests for Genkit samples" -authors = [{ name = "Google" }] -license = "Apache-2.0" -readme = "README.md" +authors = [{ name = "Google" }] +classifiers = ["Private :: Do Not Upload"] +dependencies = ["genkit", "httpx", "structlog"] +description = "Tool to run flow tests for Genkit samples" +license = "Apache-2.0" +name = "genkit-tools-sample-flows" +readme = "README.md" requires-python = ">=3.10" -dependencies = [ - "genkit", - "httpx", - "structlog", -] +version = "0.1.0" [build-system] -requires = ["hatchling"] build-backend = "hatchling.build" +requires = ["hatchling"] [tool.hatch.build.targets.wheel] packages = ["review_sample_flows.py", "run_single_flow.py"] diff --git a/releasekit.toml b/releasekit.toml index c11d5f80d4..35fc631c95 100644 --- a/releasekit.toml +++ b/releasekit.toml @@ -85,6 +85,7 @@ repo_owner = "firebase" # Python workspace # --------------------------------------------------------------------------- [workspace.py] +bootstrap_sha = "b71a3d20c74b71583edbc652e5b26117caad43f4" # py/v0.5.0 changelog = true core_package = "genkit" ecosystem = "python" @@ -94,7 +95,6 @@ plugin_dirs = ["plugins"] plugin_prefix = "genkit-plugin-" root = "py" smoke_test = true -bootstrap_sha = "b71a3d20c74b71583edbc652e5b26117caad43f4" # py/v0.5.0 tag_format = "{label}/{name}-v{version}" tool = "uv" umbrella_tag = "{label}/v{version}" @@ -111,6 +111,7 @@ exclude_publish = [ community_plugins = [ "genkit-plugin-amazon-bedrock", "genkit-plugin-anthropic", + "genkit-plugin-checks", "genkit-plugin-cloudflare-workers-ai", "genkit-plugin-cohere", "genkit-plugin-compat-oai", @@ -133,7 +134,13 @@ google_plugins = [ "genkit-plugin-google-genai", "genkit-plugin-vertex-ai", ] -internal_tools = ["conform", "genkit-plugin-dev-local-vectorstore"] +internal_tools = [ + "conform", + "genkit-plugin-dev-local-vectorstore", + "genkit-tools-model-config-test", + "genkit-tools-sample-flows", + "releasekit", +] samples = [ "dev-local-vectorstore-hello", "framework-context-demo", From baddba5cf660ab4d240d19e8de9e9e4d2d322926 Mon Sep 17 00:00:00 2001 From: Mengqin Shen Date: Sat, 14 Feb 2026 21:39:20 -0800 Subject: [PATCH 5/5] fix(py): fixed the prettier formatting error in captainhook.json --- captainhook.json | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/captainhook.json b/captainhook.json index b82b95ee3f..8c87aed790 100644 --- a/captainhook.json +++ b/captainhook.json @@ -27,15 +27,8 @@ { "run": "CaptainHook::File.BlockSecrets", "options": { - "presets": [ - "Aws", - "GitHub", - "Stripe", - "Google" - ], - "allowed": [ - "AIDAQEAAAAAAAAA" - ] + "presets": ["Aws", "GitHub", "Stripe", "Google"], + "allowed": ["AIDAQEAAAAAAAAA"] } }, { @@ -75,9 +68,7 @@ { "run": "CaptainHook::Branch.PreventPushOfFixupAndSquashCommits", "options": { - "branches-to-protect": [ - "main" - ] + "branches-to-protect": ["main"] } }, { @@ -122,4 +113,4 @@ ] } } -} \ No newline at end of file +}