diff --git a/captainhook.json b/captainhook.json index 4f3c6737ef..8c87aed790 100644 --- a/captainhook.json +++ b/captainhook.json @@ -21,7 +21,7 @@ "allow-failure": true }, "options": { - "max-size": "1M" + "max-size": "2M" } }, { diff --git a/py/bin/test_sample_flows b/py/bin/test_sample_flows index 1167d3ce3d..6db53b47e3 100755 --- a/py/bin/test_sample_flows +++ b/py/bin/test_sample_flows @@ -118,7 +118,7 @@ if [[ $# -lt 1 ]]; then # Run the flow testing tool cd "$PY_DIR" # Don't trap SIGINT here - allow user to interrupt - if uv run samples/sample-test/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then + if uv run tools/sample-flows/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then TEST_SUCCESS=true else TEST_SUCCESS=false @@ -149,6 +149,12 @@ else exit 1 fi + if [[ ! -f "$SAMPLE_DIR/src/main.py" ]] && [[ ! -f "$SAMPLE_DIR/main.py" ]]; then + echo "Error: Sample '$SAMPLE_NAME' does not have a main.py or src/main.py" + echo "Skipping..." + exit 1 + fi + echo "" echo "============================================================" echo "Testing flows in: $SAMPLE_NAME" @@ -162,7 +168,7 @@ else # Run the flow testing tool cd "$PY_DIR" # Allow Ctrl+C to interrupt - if uv run samples/sample-test/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then + if uv run tools/sample-flows/review_sample_flows.py "$SAMPLE_DIR" --output "$OUTPUT_FILE"; then echo "Flow testing completed for $SAMPLE_NAME" else echo "" diff --git a/py/pyproject.toml b/py/pyproject.toml index ad8609db24..5241abed69 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -212,6 +212,7 @@ web-fastapi-bugbot = { workspace = true } web-flask-hello = { workspace = true } web-multi-server = { workspace = true } web-short-n-long = { workspace = true } + # Core packages genkit = { workspace = true } # Plugins (alphabetical) @@ -238,11 +239,13 @@ genkit-plugin-ollama = { workspace = true } genkit-plugin-vertex-ai = { workspace = true } genkit-plugin-xai = { workspace = true } # Internal tools (private, not published) -conform = { workspace = true } +conform = { workspace = true } +genkit-tools-model-config-test = { workspace = true } +genkit-tools-sample-flows = { workspace = true } [tool.uv.workspace] -exclude = ["*/shared", "samples/sample-test", "testapps/*"] -members = ["packages/*", "plugins/*", "samples/*", "tools/conform"] +exclude = ["*/shared", "testapps/*"] +members = ["packages/*", "plugins/*", "samples/*", "tools/*"] # Ruff checks and formatting. @@ -507,6 +510,8 @@ extraPaths = [ # Tools "tools/releasekit/src", "tools/conform/src", + "tools/model-config-test", + "tools/sample-flows", ] pythonVersion = "3.10" reportMissingImports = true @@ -539,7 +544,8 @@ project_excludes = [ "build", "dist", "testapps", - "samples/sample-test/**/*.py", + "testapps", + "tools/model-config-test/**/*.py", ] project_includes = [ # Core package (source and tests) @@ -556,6 +562,7 @@ project_includes = [ "tools/releasekit/src/**/*.py", "tools/releasekit/tests/**/*.py", "tools/conform/src/**/*.py", + "tools/sample-flows/**/*.py", ] # Search path for first-party code import resolution. @@ -563,23 +570,25 @@ project_includes = [ # - plugins/mcp/tests: has a local `fakes` module for test mocks # - samples/framework-evaluator-demo: has `evaluator_demo` package with internal imports # - samples/framework-restaurant-demo/src: has internal imports (menu_ai, menu_schemas) -# - samples/sample-test: has `model_performance_test` module for imports +# - tools/model-config-test: has `model_performance_test` module for imports search-path = [ ".", "plugins/mcp/tests", "samples/framework-evaluator-demo", "samples/framework-restaurant-demo/src", - "samples/sample-test", + "samples/web-endpoints-hello", # Tools "tools/releasekit/src", "tools/releasekit", "tools/conform/src", + "tools/sample-flows", + "tools/model-config-test", ] # Ignore missing imports for namespace packages - pyrefly can't resolve PEP 420 # namespace packages but these imports work at runtime. # model_performance_test is local to sample-test (excluded from workspace). -ignore-missing-imports = ["genkit.plugins.*", "model_performance_test"] +ignore-missing-imports = ["genkit.plugins.*"] python_version = "3.10" # Treat warnings as errors. diff --git a/py/tools/model-config-test/LICENSE b/py/tools/model-config-test/LICENSE new file mode 100644 index 0000000000..2205396735 --- /dev/null +++ b/py/tools/model-config-test/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/py/samples/sample-test/README.md b/py/tools/model-config-test/README.md similarity index 71% rename from py/samples/sample-test/README.md rename to py/tools/model-config-test/README.md index 031098f098..9627c05b49 100644 --- a/py/samples/sample-test/README.md +++ b/py/tools/model-config-test/README.md @@ -1,4 +1,4 @@ -# Model Performance Testing Tool +# Model Config Testing Tool A tool to test model performance across different models and configuration variations. @@ -12,7 +12,14 @@ A tool to test model performance across different models and configuration varia Run the tool: ```bash -uv run test_model_performance.py --models googleai/gemini-2.0-flash +uv run tools/model-config-test/model_performance_test.py --models googleai/gemini-2.0-flash +``` + +Or run the web interface: + +```bash +cd py +uv run tools/model-config-test/server.py ``` ## Features diff --git a/py/samples/sample-test/model_performance_test.py b/py/tools/model-config-test/model_performance_test.py similarity index 99% rename from py/samples/sample-test/model_performance_test.py rename to py/tools/model-config-test/model_performance_test.py index 95b3c16c11..190ce3f2da 100644 --- a/py/samples/sample-test/model_performance_test.py +++ b/py/tools/model-config-test/model_performance_test.py @@ -114,7 +114,7 @@ async def discover_models_for_sample(sample_name: str) -> dict[str, Any]: logger = logging.getLogger(__name__) # Find the sample directory - samples_dir = Path(__file__).parent.parent + samples_dir = Path(__file__).parent.parent.parent / 'samples' sample_dir = samples_dir / sample_name if not sample_dir.exists(): @@ -469,7 +469,7 @@ def run_model_test( capture_output=True, text=True, timeout=timeout, - cwd=helper_script.parent, # Run from script directory (samples/sample-test) + cwd=helper_script.parent, # Run from script directory (tools/sample-flows) ) # Parse JSON output diff --git a/py/samples/sample-test/pyproject.toml b/py/tools/model-config-test/pyproject.toml similarity index 56% rename from py/samples/sample-test/pyproject.toml rename to py/tools/model-config-test/pyproject.toml index bf5d8007ac..7db5f929e8 100644 --- a/py/samples/sample-test/pyproject.toml +++ b/py/tools/model-config-test/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "tomli>=2.0.0; python_version < '3.11'", ] description = "Model Performance Testing Tool" -name = "sample-test" +name = "genkit-tools-model-config-test" readme = "README.md" requires-python = ">=3.10" version = "0.1.0" @@ -50,14 +50,14 @@ requires = ["hatchling"] packages = ["model_performance_test.py", "run_single_model_test.py"] [tool.uv.sources] -genkit = { path = "../../packages/genkit", editable = true } -genkit-plugin-amazon-bedrock = { path = "../../plugins/amazon-bedrock", editable = true } -genkit-plugin-anthropic = { path = "../../plugins/anthropic", editable = true } -genkit-plugin-deepseek = { path = "../../plugins/deepseek", editable = true } -genkit-plugin-evaluators = { path = "../../plugins/evaluators", editable = true } -genkit-plugin-google-cloud = { path = "../../plugins/google-cloud", editable = true } -genkit-plugin-google-genai = { path = "../../plugins/google-genai", editable = true } -genkit-plugin-mistral = { path = "../../plugins/mistral", editable = true } -genkit-plugin-ollama = { path = "../../plugins/ollama", editable = true } -genkit-plugin-vertex-ai = { path = "../../plugins/vertex-ai", editable = true } -genkit-plugin-xai = { path = "../../plugins/xai", editable = true } +genkit = { workspace = true } +genkit-plugin-amazon-bedrock = { workspace = true } +genkit-plugin-anthropic = { workspace = true } +genkit-plugin-deepseek = { workspace = true } +genkit-plugin-evaluators = { workspace = true } +genkit-plugin-google-cloud = { workspace = true } +genkit-plugin-google-genai = { workspace = true } +genkit-plugin-mistral = { workspace = true } +genkit-plugin-ollama = { workspace = true } +genkit-plugin-vertex-ai = { workspace = true } +genkit-plugin-xai = { workspace = true } diff --git a/py/samples/sample-test/run_single_model_test.py b/py/tools/model-config-test/run_single_model_test.py similarity index 91% rename from py/samples/sample-test/run_single_model_test.py rename to py/tools/model-config-test/run_single_model_test.py index 4ab4d15c90..93595caf9b 100644 --- a/py/samples/sample-test/run_single_model_test.py +++ b/py/tools/model-config-test/run_single_model_test.py @@ -185,7 +185,7 @@ def main() -> None: # Run test in async context import asyncio - asyncio.run( + result = asyncio.run( run_model_test( args.model_name, config, @@ -195,9 +195,18 @@ def main() -> None: ) # Output JSON result with markers - - except Exception: # noqa: S110 - intentionally silent, error handled by returning result dict - pass + print(f'---JSON_RESULT_START---\n{json.dumps(result)}\n---JSON_RESULT_END---') # noqa: T201 + + except Exception: # noqa: S110 - error is captured and reported as JSON + import traceback + + result = { + 'success': False, + 'response': None, + 'error': f'Unexpected error in test script:\n{traceback.format_exc()}', + 'timing': 0.0, + } + print(f'---JSON_RESULT_START---\n{json.dumps(result)}\n---JSON_RESULT_END---') # noqa: T201 if __name__ == '__main__': diff --git a/py/samples/sample-test/server.py b/py/tools/model-config-test/server.py similarity index 93% rename from py/samples/sample-test/server.py rename to py/tools/model-config-test/server.py index 3e21610224..72b562bd83 100644 --- a/py/samples/sample-test/server.py +++ b/py/tools/model-config-test/server.py @@ -72,7 +72,7 @@ class TestResult(BaseModel): async def discover_scenarios() -> list[Scenario]: """Discover test scenarios from samples directory.""" - samples_dir = Path(__file__).parent.parent + samples_dir = Path(__file__).parent.parent.parent / 'samples' scenarios = [] if not samples_dir.exists(): @@ -100,11 +100,14 @@ async def discover_scenarios() -> list[Scenario]: else: import tomli as tomllib # conditional dep for 3.10 - with open(pyproject_path, 'rb') as f: - data = tomllib.load(f) - project = data.get('project', {}) - name = project.get('name', name) - description = project.get('description', description) + def load_toml_data(path: Path) -> dict[str, Any]: + with open(path, 'rb') as f: + return tomllib.load(f) + + data = await asyncio.to_thread(load_toml_data, pyproject_path) + project = data.get('project', {}) + name = project.get('name', name) + description = project.get('description', description) except Exception: # noqa: S110 pass @@ -316,8 +319,11 @@ async def run_comprehensive_test( 'results': all_results, } - with open(summary_file, 'w') as f: - json.dump(summary_data, f, indent=2) + def write_summary(path: Path, data: dict[str, Any]) -> None: + with open(path, 'w') as f: + json.dump(data, f, indent=2) + + await asyncio.to_thread(write_summary, summary_file, summary_data) logging.info(f'Saved comprehensive test summary to {summary_file}') except Exception as e: diff --git a/py/samples/sample-test/static/index.html b/py/tools/model-config-test/static/index.html similarity index 100% rename from py/samples/sample-test/static/index.html rename to py/tools/model-config-test/static/index.html diff --git a/py/samples/sample-test/static/script.js b/py/tools/model-config-test/static/script.js similarity index 100% rename from py/samples/sample-test/static/script.js rename to py/tools/model-config-test/static/script.js diff --git a/py/samples/sample-test/static/style.css b/py/tools/model-config-test/static/style.css similarity index 100% rename from py/samples/sample-test/static/style.css rename to py/tools/model-config-test/static/style.css diff --git a/py/samples/sample-test/uv.lock b/py/tools/model-config-test/uv.lock similarity index 99% rename from py/samples/sample-test/uv.lock rename to py/tools/model-config-test/uv.lock index 8e306a4ff1..d50d91ada4 100644 --- a/py/samples/sample-test/uv.lock +++ b/py/tools/model-config-test/uv.lock @@ -1102,6 +1102,7 @@ version = "0.5.0" source = { editable = "../../plugins/google-cloud" } dependencies = [ { name = "genkit" }, + { name = "google-cloud-logging" }, { name = "opentelemetry-exporter-gcp-monitoring" }, { name = "opentelemetry-exporter-gcp-trace" }, { name = "strenum", marker = "python_full_version < '3.11'" }, @@ -1110,6 +1111,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "genkit", editable = "../../packages/genkit" }, + { name = "google-cloud-logging", specifier = ">=3.10.0" }, { name = "opentelemetry-exporter-gcp-monitoring", specifier = ">=1.9.0" }, { name = "opentelemetry-exporter-gcp-trace", specifier = ">=1.9.0" }, { name = "strenum", marker = "python_full_version < '3.11'", specifier = ">=0.4.15" }, @@ -1285,6 +1287,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/e8/f317dc96c9c73846dd3e4d16691cc5f248801f46354d9d57f2c67fd67413/google_cloud_aiplatform-1.136.0-py2.py3-none-any.whl", hash = "sha256:5c829f002b7b673dcd0e718f55cc0557b571bd10eb5cdb7882d72916cfbf8c0e", size = 8203924, upload-time = "2026-02-04T16:28:10.343Z" }, ] +[[package]] +name = "google-cloud-appengine-logging" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "grpcio" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/38/89317773c64b5a7e9b56b9aecb2e39ac02d8d6d09fb5b276710c6892e690/google_cloud_appengine_logging-1.8.0.tar.gz", hash = "sha256:84b705a69e4109fc2f68dfe36ce3df6a34d5c3d989eee6d0ac1b024dda0ba6f5", size = 18071, upload-time = "2026-01-15T13:14:40.024Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/66/4a9be8afb1d0bf49472478cec20fefe4f4cb3a6e67be2231f097041e7339/google_cloud_appengine_logging-1.8.0-py3-none-any.whl", hash = "sha256:a4ce9ce94a9fd8c89ed07fa0b06fcf9ea3642f9532a1be1a8c7b5f82c0a70ec6", size = 18380, upload-time = "2026-01-09T14:52:58.154Z" }, +] + +[[package]] +name = "google-cloud-audit-log" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/d2/ad96950410f8a05e921a6da2e1a6ba4aeca674bbb5dda8200c3c7296d7ad/google_cloud_audit_log-0.4.0.tar.gz", hash = "sha256:8467d4dcca9f3e6160520c24d71592e49e874838f174762272ec10e7950b6feb", size = 44682, upload-time = "2025-10-17T02:33:44.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/25/532886995f11102ad6de290496de5db227bd3a73827702445928ad32edcb/google_cloud_audit_log-0.4.0-py3-none-any.whl", hash = "sha256:6b88e2349df45f8f4cc0993b687109b1388da1571c502dc1417efa4b66ec55e0", size = 44890, upload-time = "2025-10-17T02:30:55.11Z" }, +] + [[package]] name = "google-cloud-bigquery" version = "3.40.0" @@ -1332,6 +1363,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/99/2a627c8ea7ae72a686dda8bf2b79747362b425c237d2729eb76bcee55a25/google_cloud_firestore-2.23.0-py3-none-any.whl", hash = "sha256:19f2326cb466b0d52aed9fabbd89758be431f6ce18c422966cfdb8326b424314", size = 411195, upload-time = "2026-01-14T23:50:52.825Z" }, ] +[[package]] +name = "google-cloud-logging" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "google-cloud-appengine-logging" }, + { name = "google-cloud-audit-log" }, + { name = "google-cloud-core" }, + { name = "grpc-google-iam-v1" }, + { name = "opentelemetry-api" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/47/31ef0261802fe8b37c221392e1d6ff01d30b03dce5e20e77fc7d57ddf8a3/google_cloud_logging-3.13.0.tar.gz", hash = "sha256:3aae0573b1a1a4f59ecdf4571f4e7881b5823bd129fe469561c1c49a7fa8a4c1", size = 290169, upload-time = "2025-12-16T14:11:07.345Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/5a/778dca2e375171af4085554cb3bc643627717a7e4e1539842ced3afd6ec4/google_cloud_logging-3.13.0-py3-none-any.whl", hash = "sha256:f215e1c76ee29239c6cacf02443dffa985663c74bf47c9818854694805c6019f", size = 230518, upload-time = "2025-12-16T14:11:05.894Z" }, +] + [[package]] name = "google-cloud-monitoring" version = "2.29.1" diff --git a/py/tools/releasekit/pyproject.toml b/py/tools/releasekit/pyproject.toml index ff671222c2..37f15e158a 100644 --- a/py/tools/releasekit/pyproject.toml +++ b/py/tools/releasekit/pyproject.toml @@ -32,6 +32,7 @@ classifiers = [ "Topic :: Software Development :: Build Tools", "Topic :: Software Development :: Libraries :: Python Modules", "Typing :: Typed", + "Private :: Do Not Upload", ] dependencies = [ "aiofiles>=24.1.0", # Non-blocking file I/O for async Workspace protocol diff --git a/py/tools/sample-flows/LICENSE b/py/tools/sample-flows/LICENSE new file mode 100644 index 0000000000..2205396735 --- /dev/null +++ b/py/tools/sample-flows/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/py/tools/sample-flows/README.md b/py/tools/sample-flows/README.md new file mode 100644 index 0000000000..48b75654f0 --- /dev/null +++ b/py/tools/sample-flows/README.md @@ -0,0 +1,30 @@ +# Sample Flow Testing Tool + +This directory contains scripts for reviewing and testing Genkit flows within samples. + +## Files + +- `review_sample_flows.py`: Iterates through all flows in a given sample directory, runs them with heuristic inputs, and generates a report. +- `run_single_flow.py`: Helper script to run a single flow in isolation (used by `review_sample_flows.py`). + +## Usage + +This tool is typically run via the `py/bin/test_sample_flows` script: + +```bash +# Test a specific sample +py/bin/test_sample_flows provider-google-genai-hello +``` + +Or manually: + +```bash +# Run from the repository root (py/) +uv run tools/sample-flows/review_sample_flows.py samples/provider-google-genai-hello +``` + +## Output + +The tool generates a text report (e.g., `flow_review_results.txt`) detailing which flows passed or failed, along with their outputs or error messages. + +**Note:** The `test_sample_flows` script automatically skips samples that do not have a standard `main.py` entry point (e.g., `framework-evaluator-demo`), preventing execution errors. diff --git a/py/tools/sample-flows/pyproject.toml b/py/tools/sample-flows/pyproject.toml new file mode 100644 index 0000000000..59306f93b2 --- /dev/null +++ b/py/tools/sample-flows/pyproject.toml @@ -0,0 +1,36 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +[project] +authors = [{ name = "Google" }] +classifiers = ["Private :: Do Not Upload"] +dependencies = ["genkit", "httpx", "structlog"] +description = "Tool to run flow tests for Genkit samples" +license = "Apache-2.0" +name = "genkit-tools-sample-flows" +readme = "README.md" +requires-python = ">=3.10" +version = "0.1.0" + +[build-system] +build-backend = "hatchling.build" +requires = ["hatchling"] + +[tool.hatch.build.targets.wheel] +packages = ["review_sample_flows.py", "run_single_flow.py"] + +[tool.uv.sources] +genkit = { workspace = true } diff --git a/py/samples/sample-test/review_sample_flows.py b/py/tools/sample-flows/review_sample_flows.py similarity index 99% rename from py/samples/sample-test/review_sample_flows.py rename to py/tools/sample-flows/review_sample_flows.py index e813a22c9c..91d11b92b2 100644 --- a/py/samples/sample-test/review_sample_flows.py +++ b/py/tools/sample-flows/review_sample_flows.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,26 +29,26 @@ import importlib.util import json import logging - -logging.getLogger().setLevel(logging.ERROR) -logging.getLogger('asyncio').setLevel(logging.ERROR) -logging.getLogger('httpx').setLevel(logging.ERROR) -logging.getLogger('httpcore').setLevel(logging.ERROR) import platform import re -import time -import warnings - -warnings.filterwarnings('ignore') import subprocess # noqa: S404 import sys +import time import traceback +import warnings from pathlib import Path from typing import Any from genkit.core.action import ActionKind from genkit.types import Media +logging.getLogger().setLevel(logging.ERROR) +logging.getLogger('asyncio').setLevel(logging.ERROR) +logging.getLogger('httpx').setLevel(logging.ERROR) +logging.getLogger('httpcore').setLevel(logging.ERROR) + +warnings.filterwarnings('ignore') + def open_file(path: str) -> None: """Open a file with the default system application.""" diff --git a/py/samples/sample-test/run_single_flow.py b/py/tools/sample-flows/run_single_flow.py similarity index 99% rename from py/samples/sample-test/run_single_flow.py rename to py/tools/sample-flows/run_single_flow.py index be9fde97eb..53311ea1ef 100644 --- a/py/samples/sample-test/run_single_flow.py +++ b/py/tools/sample-flows/run_single_flow.py @@ -16,6 +16,7 @@ # SPDX-License-Identifier: Apache-2.0 # pyrefly: ignore-file +# flake8: noqa: ASYNC240 """Helper script to run a single Genkit flow in isolation. diff --git a/py/uv.lock b/py/uv.lock index 7cfa269dd8..0e9dc778db 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -45,6 +45,8 @@ members = [ "genkit-plugin-ollama", "genkit-plugin-vertex-ai", "genkit-plugin-xai", + "genkit-tools-model-config-test", + "genkit-tools-sample-flows", "genkit-workspace", "provider-amazon-bedrock-hello", "provider-anthropic-hello", @@ -70,6 +72,7 @@ members = [ "provider-vertex-ai-vector-search-bigquery", "provider-vertex-ai-vector-search-firestore", "provider-xai-hello", + "releasekit", "web-endpoints-hello", "web-fastapi-bugbot", "web-flask-hello", @@ -2726,6 +2729,96 @@ requires-dist = [ { name = "xai-sdk", specifier = ">=0.0.1" }, ] +[[package]] +name = "genkit-tools-model-config-test" +version = "0.1.0" +source = { editable = "tools/model-config-test" } +dependencies = [ + { name = "datamodel-code-generator" }, + { name = "fastapi" }, + { name = "genkit" }, + { name = "genkit-plugin-amazon-bedrock" }, + { name = "genkit-plugin-anthropic" }, + { name = "genkit-plugin-deepseek" }, + { name = "genkit-plugin-evaluators" }, + { name = "genkit-plugin-google-cloud" }, + { name = "genkit-plugin-google-genai" }, + { name = "genkit-plugin-mistral" }, + { name = "genkit-plugin-ollama" }, + { name = "genkit-plugin-vertex-ai" }, + { name = "genkit-plugin-xai" }, + { name = "grpcio" }, + { name = "grpcio-reflection" }, + { name = "gunicorn" }, + { name = "hypercorn" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-asgi" }, + { name = "opentelemetry-instrumentation-fastapi" }, + { name = "opentelemetry-instrumentation-grpc" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic-settings" }, + { name = "quart" }, + { name = "secure" }, + { name = "structlog" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "uvicorn" }, + { name = "uvloop" }, +] + +[package.metadata] +requires-dist = [ + { name = "datamodel-code-generator" }, + { name = "fastapi" }, + { name = "genkit", editable = "packages/genkit" }, + { name = "genkit-plugin-amazon-bedrock", editable = "plugins/amazon-bedrock" }, + { name = "genkit-plugin-anthropic", editable = "plugins/anthropic" }, + { name = "genkit-plugin-deepseek", editable = "plugins/deepseek" }, + { name = "genkit-plugin-evaluators", editable = "plugins/evaluators" }, + { name = "genkit-plugin-google-cloud", editable = "plugins/google-cloud" }, + { name = "genkit-plugin-google-genai", editable = "plugins/google-genai" }, + { name = "genkit-plugin-mistral", editable = "plugins/mistral" }, + { name = "genkit-plugin-ollama", editable = "plugins/ollama" }, + { name = "genkit-plugin-vertex-ai", editable = "plugins/vertex-ai" }, + { name = "genkit-plugin-xai", editable = "plugins/xai" }, + { name = "grpcio" }, + { name = "grpcio-reflection" }, + { name = "gunicorn" }, + { name = "hypercorn" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-asgi" }, + { name = "opentelemetry-instrumentation-fastapi" }, + { name = "opentelemetry-instrumentation-grpc" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic-settings" }, + { name = "quart" }, + { name = "secure" }, + { name = "structlog" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.0" }, + { name = "uvicorn" }, + { name = "uvloop" }, +] + +[[package]] +name = "genkit-tools-sample-flows" +version = "0.1.0" +source = { editable = "tools/sample-flows" } +dependencies = [ + { name = "genkit" }, + { name = "httpx" }, + { name = "structlog" }, +] + +[package.metadata] +requires-dist = [ + { name = "genkit", editable = "packages/genkit" }, + { name = "httpx" }, + { name = "structlog" }, +] + [[package]] name = "genkit-workspace" version = "0.1.0" @@ -8126,6 +8219,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, ] +[[package]] +name = "releasekit" +version = "0.1.0" +source = { editable = "tools/releasekit" } +dependencies = [ + { name = "aiofiles" }, + { name = "argcomplete" }, + { name = "diagnostic" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "rich" }, + { name = "rich-argparse" }, + { name = "strenum", marker = "python_full_version < '3.11'" }, + { name = "structlog" }, + { name = "tomlkit" }, +] + +[package.optional-dependencies] +tracing = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiofiles", specifier = ">=24.1.0" }, + { name = "argcomplete", specifier = ">=3.0.0" }, + { name = "diagnostic", specifier = ">=3.0.0" }, + { name = "httpx", specifier = ">=0.27.0" }, + { name = "jinja2", specifier = ">=3.1.0" }, + { name = "opentelemetry-api", marker = "extra == 'tracing'", specifier = ">=1.20.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'tracing'", specifier = ">=1.20.0" }, + { name = "packaging", specifier = ">=24.0" }, + { name = "rich", specifier = ">=13.0.0" }, + { name = "rich-argparse", specifier = ">=1.6.0" }, + { name = "strenum", marker = "python_full_version < '3.11'", specifier = ">=0.4.15" }, + { name = "structlog", specifier = ">=25.1.0" }, + { name = "tomlkit", specifier = ">=0.13.0" }, +] +provides-extras = ["tracing"] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", specifier = ">=0.25.0" }, + { name = "pytest-cov", specifier = ">=7.0.0" }, +] + [[package]] name = "requests" version = "2.32.5" diff --git a/releasekit.toml b/releasekit.toml index c11d5f80d4..35fc631c95 100644 --- a/releasekit.toml +++ b/releasekit.toml @@ -85,6 +85,7 @@ repo_owner = "firebase" # Python workspace # --------------------------------------------------------------------------- [workspace.py] +bootstrap_sha = "b71a3d20c74b71583edbc652e5b26117caad43f4" # py/v0.5.0 changelog = true core_package = "genkit" ecosystem = "python" @@ -94,7 +95,6 @@ plugin_dirs = ["plugins"] plugin_prefix = "genkit-plugin-" root = "py" smoke_test = true -bootstrap_sha = "b71a3d20c74b71583edbc652e5b26117caad43f4" # py/v0.5.0 tag_format = "{label}/{name}-v{version}" tool = "uv" umbrella_tag = "{label}/v{version}" @@ -111,6 +111,7 @@ exclude_publish = [ community_plugins = [ "genkit-plugin-amazon-bedrock", "genkit-plugin-anthropic", + "genkit-plugin-checks", "genkit-plugin-cloudflare-workers-ai", "genkit-plugin-cohere", "genkit-plugin-compat-oai", @@ -133,7 +134,13 @@ google_plugins = [ "genkit-plugin-google-genai", "genkit-plugin-vertex-ai", ] -internal_tools = ["conform", "genkit-plugin-dev-local-vectorstore"] +internal_tools = [ + "conform", + "genkit-plugin-dev-local-vectorstore", + "genkit-tools-model-config-test", + "genkit-tools-sample-flows", + "releasekit", +] samples = [ "dev-local-vectorstore-hello", "framework-context-demo",