From 71eefe57d92d8d3fdfbd0fd9e51b32278223fd0d Mon Sep 17 00:00:00 2001 From: etserend Date: Thu, 30 Jul 2026 18:41:17 -0500 Subject: [PATCH] fix(experiment): remove deprecated job_id parameter from monitor_progress() (HYBIM-931) Co-Authored-By: Claude Opus 4.7 --- CHANGELOG.md | 728 ++++++++++++++++++++++++++++++ src/splunk_ao/experiment.py | 13 - tests/test_experiment_progress.py | 11 - 3 files changed, 728 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fbf795..0f2369d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## Unreleased +### Breaking Changes + +- **`monitor_progress()` `job_id` parameter removed** (HYBIM-931): The deprecated + `job_id` keyword argument of `Experiment.monitor_progress()` has been fully removed. + Callers passing `job_id=` must remove that argument. + +- `SplunkAOLogger.flush()` and `async_flush()` now return `None` and only drain + completed spans already queued for OTLP export. They no longer conclude open + spans or return uploaded proprietary trace objects. + +- **`OpenAIGalileo` renamed to `OpenAISplunkAO`** (HYBIM-726): The OpenAI wrapper class in + `splunk_ao.openai` has been renamed as part of the Galileo → Splunk AO rebrand. Any code + directly importing this class by name must update to + `from splunk_ao.openai import OpenAISplunkAO`. + ### Added - Added deployment-aware configuration and authentication for Splunk @@ -40,3 +55,716 @@ - Exporters expose bounded, read-only receiver-acknowledgement health. Health is unknown before an acknowledgement and after ordinary transport or non-2xx failures; it is not a delivery guarantee. + +### Security + +- **deps (VULN-88813)**: Raise the optional `starlette` floor to `>=1.3.1` (from `>=0.27.0`) to remediate urlencoded form limit bypass in `request.form()`. Starlette 1.x removes APIs deprecated since 0.x; consumers of the `middleware`, `openai`, or `crewai` extras who rely on removed Starlette 0.x APIs may need to update custom middleware. + +### Features + +- **New `generated_output` field**: Add `generated_output` field to `DatasetRecord` for storing model-generated outputs separately from ground truth. This allows you to track both the expected output (ground truth) and the actual model output in the same dataset record. In the UI, this field is displayed as "Generated Output". + + Example: + ```python + from galileo.schema.datasets import DatasetRecord + + record = DatasetRecord( + input="What is 2+2?", + output="4", # Ground truth + generated_output="The answer is 4" # Model-generated output + ) + ``` + +- **Ground Truth naming support**: The existing `output` field is now displayed as "Ground Truth" in the Galileo UI for better clarity. The SDK supports both `output` and `ground_truth` field names when creating records - both are normalized to `output` internally, ensuring full backward compatibility. You can use either field name, and access the value via the `ground_truth` property. + + Example: + ```python + from galileo.schema.datasets import DatasetRecord + + # Using 'output' (backward compatible) + record1 = DatasetRecord(input="What is 2+2?", output="4") + assert record1.ground_truth == "4" # Property accessor + + # Using 'ground_truth' (new recommended way) + record2 = DatasetRecord(input="What is 2+2?", ground_truth="4") + assert record2.output == "4" # Normalized internally + assert record2.ground_truth == "4" # Property accessor + ``` + + +## v0.10.0 (2025-05-29) + +### Bug Fixes + +- Langchain on_chain_start with_kwargs name and serialised is none + ([#138](https://github.com/rungalileo/galileo-python/pull/138), + [`132955c`](https://github.com/rungalileo/galileo-python/commit/132955cd88f369e99c203262dfd77c394a4e02a7)) + +### Chores + +- **deps**: Bump langchain-core from 0.3.40 to 0.3.58 + ([#122](https://github.com/rungalileo/galileo-python/pull/122), + [`e2e6a49`](https://github.com/rungalileo/galileo-python/commit/e2e6a49cca0657ebd6483a3d361505422fc754ca)) + +### Features + +- Add codeflash optimizer on our repo + ([#151](https://github.com/rungalileo/galileo-python/pull/151), + [`de1897b`](https://github.com/rungalileo/galileo-python/commit/de1897bf9aa46aa2fba0f9f7870a27ce766985b5)) + +- Enable setting session ID ([#139](https://github.com/rungalileo/galileo-python/pull/139), + [`e646df3`](https://github.com/rungalileo/galileo-python/commit/e646df34606a3949543364c15c44c36b5b697bdb)) + +This changes the `start_session` function to return the session ID, and adds a new `set_session` + function. + +- **langchain**: Implement duration_ns metric for langchain callback async handler + ([#150](https://github.com/rungalileo/galileo-python/pull/150), + [`eed7a81`](https://github.com/rungalileo/galileo-python/commit/eed7a81f89e4e15801fa95e65229d08ea9824cab)) + +- **langchain**: Implement duration_ns metric for langchain callback handler + ([#145](https://github.com/rungalileo/galileo-python/pull/145), + [`55f3cdf`](https://github.com/rungalileo/galileo-python/commit/55f3cdf63cbc254f4d8804ef61081c877b4c61b7)) + + +## v0.9.2 (2025-05-22) + +### Bug Fixes + +- Creating empty dataset ([#134](https://github.com/rungalileo/galileo-python/pull/134), + [`3b13f52`](https://github.com/rungalileo/galileo-python/commit/3b13f522de20ba977915a310e1b7aea184b867f5)) + +- Runtimewarning: coroutine AsyncMockMixin._execute_mock_call was never awaited + ([#135](https://github.com/rungalileo/galileo-python/pull/135), + [`bd76cff`](https://github.com/rungalileo/galileo-python/commit/bd76cffb4098f38e064c720305ddd2f2bd368bed)) + +- Updating Langchain handler issues ([#118](https://github.com/rungalileo/galileo-python/pull/118), + [`4304d16`](https://github.com/rungalileo/galileo-python/commit/4304d16d727ef797103e1a8efcb6c7d2531021f6)) + +### Chores + +- **deps-dev**: Bump mkdocs-material from 9.6.13 to 9.6.14 + ([#136](https://github.com/rungalileo/galileo-python/pull/136), + [`71d6676`](https://github.com/rungalileo/galileo-python/commit/71d667610930967b77fa687cc691a1a9cc735322)) + +- **release**: V0.9.2 + ([`12ad78c`](https://github.com/rungalileo/galileo-python/commit/12ad78c78a55e2c1db0f7bc85ffcbe307b15337a)) + +Automatically generated by python-semantic-release + + +## v0.9.1 (2025-05-15) + +### Bug Fixes + +- Reverting a change from a previous context manager fix + ([#133](https://github.com/rungalileo/galileo-python/pull/133), + [`b3bd345`](https://github.com/rungalileo/galileo-python/commit/b3bd34554f4094fe6dc178242bb2af3f8218efcf)) + +### Chores + +- **deps-dev**: Bump mkdocs-material from 9.6.12 to 9.6.13 + ([#131](https://github.com/rungalileo/galileo-python/pull/131), + [`f44be0c`](https://github.com/rungalileo/galileo-python/commit/f44be0ca98e7468d207fe0b9eafae38d325ffd60)) + +- **deps-dev**: Bump pytest-asyncio from 0.25.3 to 0.26.0 + ([#129](https://github.com/rungalileo/galileo-python/pull/129), + [`43e2af7`](https://github.com/rungalileo/galileo-python/commit/43e2af726be1b35c3594f987598bae2242f911f6)) + +- **release**: V0.9.1 + ([`b80a218`](https://github.com/rungalileo/galileo-python/commit/b80a218735d3092c0748c1eb348d34a100363b63)) + +Automatically generated by python-semantic-release + +### Continuous Integration + +- Run tests on all OSes ([#127](https://github.com/rungalileo/galileo-python/pull/127), + [`d6f3d1b`](https://github.com/rungalileo/galileo-python/commit/d6f3d1bf494d7568abe776b87c7f4be4d750ee9c)) + + +## v0.9.0 (2025-05-10) + +### Bug Fixes + +- Using context var stacks for saving context with nested _call_ s + ([#126](https://github.com/rungalileo/galileo-python/pull/126), + [`42c1424`](https://github.com/rungalileo/galileo-python/commit/42c14245d7099f1fb73c93c5d2e19b94d8061a12)) + +### Chores + +- **deps**: Bump codecov/codecov-action from 5.4.0 to 5.4.2 + ([#119](https://github.com/rungalileo/galileo-python/pull/119), + [`ba53ebf`](https://github.com/rungalileo/galileo-python/commit/ba53ebf3976e6f92a415611d088c368d2262e10c)) + +- **deps**: Bump openai-agents from 0.0.7 to 0.0.12 + ([#123](https://github.com/rungalileo/galileo-python/pull/123), + [`0150eee`](https://github.com/rungalileo/galileo-python/commit/0150eee62b4763fc9d3acf6e233a8b9e8d7cbe31)) + +- **deps-dev**: Bump mkdocs-material from 9.6.5 to 9.6.12 + ([#125](https://github.com/rungalileo/galileo-python/pull/125), + [`707149a`](https://github.com/rungalileo/galileo-python/commit/707149a4614709548bf1714f770099a44f9fe9a7)) + +- **deps-dev**: Bump mkdocstrings from 0.27.0 to 0.29.1 + ([#124](https://github.com/rungalileo/galileo-python/pull/124), + [`bdb2c47`](https://github.com/rungalileo/galileo-python/commit/bdb2c4736fac595640b5723d62f90738726503fd)) + +- **deps-dev**: Bump pytest from 8.3.4 to 8.3.5 + ([#121](https://github.com/rungalileo/galileo-python/pull/121), + [`7e5ab19`](https://github.com/rungalileo/galileo-python/commit/7e5ab19c8e43775ce0e9d3085637b27781635afd)) + +- **release**: V0.9.0 + ([`eeb6ddf`](https://github.com/rungalileo/galileo-python/commit/eeb6ddffa9ab2292142a603d0eadb02f71bee603)) + +Automatically generated by python-semantic-release + +### Features + +- Local scorers for local runner experiments + ([#116](https://github.com/rungalileo/galileo-python/pull/116), + [`4f7d8e0`](https://github.com/rungalileo/galileo-python/commit/4f7d8e02e40a2513b1e2a7b3a35a200d7db799e7)) + +- Session management ([#120](https://github.com/rungalileo/galileo-python/pull/120), + [`fed6f65`](https://github.com/rungalileo/galileo-python/commit/fed6f65c7f1887bbf1e8bc89388d064a57a9cc72)) + + +## v0.8.1 (2025-04-30) + +### Chores + +- Restrict galileo-core version to only float forward for patch releases + ([#115](https://github.com/rungalileo/galileo-python/pull/115), + [`37a4c36`](https://github.com/rungalileo/galileo-python/commit/37a4c3670fe6c0e23d17e3238228bb3fc9d04919)) + +- **release**: V0.8.1 + ([`a308655`](https://github.com/rungalileo/galileo-python/commit/a308655d77a6f2beaad98c8c515b97f264db84f5)) + +Automatically generated by python-semantic-release + + +## v0.8.0 (2025-04-30) + +### Bug Fixes + +- Use correct logging schemas from `galileo_core.schemas.logging` + ([#114](https://github.com/rungalileo/galileo-python/pull/114), + [`fa387d3`](https://github.com/rungalileo/galileo-python/commit/fa387d3698dac559ad56e6d5b0e4a21ee82554d2)) + +### Chores + +- Pulling the latest openapi.yaml and regenerating the API client + ([#112](https://github.com/rungalileo/galileo-python/pull/112), + [`bfa38fb`](https://github.com/rungalileo/galileo-python/commit/bfa38fb9a4ead7a0df8dc6a02e77945f4d1578d7)) + +- **deps**: Bump h11 from 0.14.0 to 0.16.0 + ([#113](https://github.com/rungalileo/galileo-python/pull/113), + [`1336bef`](https://github.com/rungalileo/galileo-python/commit/1336bef29f32b36e1663f77051841e6004699676)) + +- **release**: V0.8.0 + ([`3335cb9`](https://github.com/rungalileo/galileo-python/commit/3335cb92c6f4904fdc5c05f40f37abd3657e264d)) + +Automatically generated by python-semantic-release + +### Documentation + +- Add supported py versions, refactor urls + ([#111](https://github.com/rungalileo/galileo-python/pull/111), + [`d6e7169`](https://github.com/rungalileo/galileo-python/commit/d6e7169a05e4c59115923b0d7a0623e76336f385)) + +### Features + +- Enable mypy on ci ([#108](https://github.com/rungalileo/galileo-python/pull/108), + [`c445a09`](https://github.com/rungalileo/galileo-python/commit/c445a09587c701ac65cbfce90406e53db3a2b0fb)) + + +## v0.7.0 (2025-04-17) + +### Bug Fixes + +- Adding list_prompt_templates ([#102](https://github.com/rungalileo/galileo-python/pull/102), + [`5814c90`](https://github.com/rungalileo/galileo-python/commit/5814c90fadf1a0c534cf168764d4871940ad20e7)) + +- Error occurred during execution: shutdown: 'GalileoTracingProcessor' object has no attribute + '_commit' ([#105](https://github.com/rungalileo/galileo-python/pull/105), + [`e2ece22`](https://github.com/rungalileo/galileo-python/commit/e2ece2210c453c5058bfe2d3b4164be4644beeef)) + +- Fix functionality to add new rows to dataset + ([#103](https://github.com/rungalileo/galileo-python/pull/103), + [`5f2d633`](https://github.com/rungalileo/galileo-python/commit/5f2d6337c440af83ae54d9809dc0a510833fc58d)) + +- Fixing the OpenAI wrapper when it's used in an active trace + ([#109](https://github.com/rungalileo/galileo-python/pull/109), + [`10bb6f6`](https://github.com/rungalileo/galileo-python/commit/10bb6f6b46dffb9089f4ec4bd2fce701c5433ab5)) + +### Chores + +- **release**: V0.7.0 + ([`76ab22d`](https://github.com/rungalileo/galileo-python/commit/76ab22d9f8b6a3466da0ec2264225e3fa7e7b1df)) + +Automatically generated by python-semantic-release + +### Features + +- Bubble up openai status_code ([#107](https://github.com/rungalileo/galileo-python/pull/107), + [`8074ea3`](https://github.com/rungalileo/galileo-python/commit/8074ea352501e4f8e80d1569eeec1cf4b9f67068)) + +### Testing + +- Bump codecov target ([#106](https://github.com/rungalileo/galileo-python/pull/106), + [`04ee6eb`](https://github.com/rungalileo/galileo-python/commit/04ee6ebe64930a9b8f9dabd03bfb227079e99011)) + + +## v0.6.0 (2025-04-10) + +### Bug Fixes + +- Adding tools sent to the chat model in the Langchain callback handlers + ([#95](https://github.com/rungalileo/galileo-python/pull/95), + [`125de12`](https://github.com/rungalileo/galileo-python/commit/125de12b889975250000dfb34d522d498b4ba668)) + +Co-authored-by: Andrii Soldatenko + +- Json datasets are not being passed into runner functions correctly + ([#100](https://github.com/rungalileo/galileo-python/pull/100), + [`e833481`](https://github.com/rungalileo/galileo-python/commit/e8334811084d0e74fb98341509d221851dae20aa)) + +- Updating README snippets to point to the correct OpenAI client wrapper + ([#92](https://github.com/rungalileo/galileo-python/pull/92), + [`ffd2a68`](https://github.com/rungalileo/galileo-python/commit/ffd2a68d5db198c76f2add24d46e46a805f426d9)) + +- Updating the Langchain handler retriever, tool, async chain callbacks + ([#101](https://github.com/rungalileo/galileo-python/pull/101), + [`ce29a91`](https://github.com/rungalileo/galileo-python/commit/ce29a91bb581c78fc01b1ee97ab2f2b6a043b17a)) + +- Use `Message` and `MessageRole` models from `galileo-core` + ([#99](https://github.com/rungalileo/galileo-python/pull/99), + [`c9ad656`](https://github.com/rungalileo/galileo-python/commit/c9ad656a702c1b1950c349fc43ca839c4e5cd799)) + +### Chores + +- Enum name normalization ([#91](https://github.com/rungalileo/galileo-python/pull/91), + [`e084dac`](https://github.com/rungalileo/galileo-python/commit/e084daccc3ad46318ac4a02d3adc50fd3e2d5781)) + +- Increase codecov target, since we alredy > 75% + ([#93](https://github.com/rungalileo/galileo-python/pull/93), + [`4a09189`](https://github.com/rungalileo/galileo-python/commit/4a091897edeff3aa0d0311101c5e57f9ebc97071)) + +- Revert "Enum name normalization" ([#98](https://github.com/rungalileo/galileo-python/pull/98), + [`508af9e`](https://github.com/rungalileo/galileo-python/commit/508af9e8ab9040ca59d3986b1368d3dc0111f6d9)) + +- **release**: V0.6.0 + ([`ff166d8`](https://github.com/rungalileo/galileo-python/commit/ff166d86fa0786401172369e6ff5f4ef6993fadd)) + +Automatically generated by python-semantic-release + +### Features + +- Add dataset version methods ([#94](https://github.com/rungalileo/galileo-python/pull/94), + [`4081828`](https://github.com/rungalileo/galileo-python/commit/408182825fde179f4a40b1ca3114116b60fa5d7a)) + +- Allowing custom hosted scorers to be used + ([#87](https://github.com/rungalileo/galileo-python/pull/87), + [`e908d87`](https://github.com/rungalileo/galileo-python/commit/e908d8707c65ee1e11b0868ebfc67479b3cef59a)) + +Co-authored-by: Andrii Soldatenko + +- Error if a non existent metric is specified + ([#97](https://github.com/rungalileo/galileo-python/pull/97), + [`93a4914`](https://github.com/rungalileo/galileo-python/commit/93a4914399cca4013af2e1e77cba1ad5a9478f88)) + +- Improve error handling for create_dataset + ([#96](https://github.com/rungalileo/galileo-python/pull/96), + [`0dda7ba`](https://github.com/rungalileo/galileo-python/commit/0dda7baf67ec1f86d40a71794279b3e860255d23)) + + +## v0.5.0 (2025-04-04) + +### Chores + +- **release**: V0.5.0 + ([`2203995`](https://github.com/rungalileo/galileo-python/commit/2203995932c767a8df1eabc8320d4b083a734299)) + +Automatically generated by python-semantic-release + +### Features + +- **integration**: Add support for openai agents (with tree) + ([#88](https://github.com/rungalileo/galileo-python/pull/88), + [`78b0c56`](https://github.com/rungalileo/galileo-python/commit/78b0c56681065fc6db7a3944aa7a5ae5f43ba3f5)) + +Co-authored-by: Andrii Soldatenko + + +## v0.4.0 (2025-04-04) + +### Chores + +- **release**: V0.4.0 + ([`8d700bc`](https://github.com/rungalileo/galileo-python/commit/8d700bcc321e3476b132e6b5a65cad2a19365f0a)) + +Automatically generated by python-semantic-release + +### Features + +- Langchain async callback handler ([#89](https://github.com/rungalileo/galileo-python/pull/89), + [`ea8d4f8`](https://github.com/rungalileo/galileo-python/commit/ea8d4f81ecf9d26fb6aec849354bd461736195c9)) + +Co-authored-by: Andrii Soldatenko + + +## v0.3.0 (2025-04-02) + +### Bug Fixes + +- Ierror handling in Experiments.get ([#84](https://github.com/rungalileo/galileo-python/pull/84), + [`0d36b8c`](https://github.com/rungalileo/galileo-python/commit/0d36b8c86d8034da4a4daf7e763136cc9c9b2e69)) + +- Improve error message when galileo api key is empty or invalid + ([#82](https://github.com/rungalileo/galileo-python/pull/82), + [`31593c4`](https://github.com/rungalileo/galileo-python/commit/31593c4b82acaf76e41ea9cab5361b6b6e634ef4)) + +- Make No traces to flush as info ([#83](https://github.com/rungalileo/galileo-python/pull/83), + [`babcd2f`](https://github.com/rungalileo/galileo-python/commit/babcd2fd40f2517310249c34fb1ab1bd02c69c9d)) + +- Making the run_experiment return object consistent + ([#86](https://github.com/rungalileo/galileo-python/pull/86), + [`4a3059f`](https://github.com/rungalileo/galileo-python/commit/4a3059f13fcfbd06de8b8f607dfd980edaa27d4d)) + +### Chores + +- **release**: V0.3.0 + ([`8fee183`](https://github.com/rungalileo/galileo-python/commit/8fee183ffa1d4f49ed582a50acfc255400db8ea7)) + +Automatically generated by python-semantic-release + +### Features + +- Add ability to disable logger ([#68](https://github.com/rungalileo/galileo-python/pull/68), + [`b417bd6`](https://github.com/rungalileo/galileo-python/commit/b417bd6d78ab2468ce43429a6ea9524f63d8285f)) + +- Experiments name should not be re used + ([#79](https://github.com/rungalileo/galileo-python/pull/79), + [`483367b`](https://github.com/rungalileo/galileo-python/commit/483367b213cafcb963fdac54ca7dfe0557ba1aca)) + + +## v0.2.4 (2025-03-28) + +### Bug Fixes + +- Improve error handling and print better error when create prompt template fails + ([#81](https://github.com/rungalileo/galileo-python/pull/81), + [`8839f8b`](https://github.com/rungalileo/galileo-python/commit/8839f8b037e342b037f58916fe99a81273ce4ef1)) + +- Set correct openai status code for successful run + ([#80](https://github.com/rungalileo/galileo-python/pull/80), + [`6d8224f`](https://github.com/rungalileo/galileo-python/commit/6d8224f2478855f273afb6be8e16191bb2e6bad2)) + +- Update explanation for Context Relevance + ([#77](https://github.com/rungalileo/galileo-python/pull/77), + [`43e8230`](https://github.com/rungalileo/galileo-python/commit/43e8230730abdfb557a705ea8867ed402ce36e92)) + +### Chores + +- **deps**: Bump `galileo-core` to v3.19.0 + ([#78](https://github.com/rungalileo/galileo-python/pull/78), + [`8391fb3`](https://github.com/rungalileo/galileo-python/commit/8391fb33e81f6ae764e79b19ce49336b7bac722b)) + +- **release**: V0.2.4 + ([`4e68e33`](https://github.com/rungalileo/galileo-python/commit/4e68e33fc738faf6953909f891c7aa99f48e364a)) + +Automatically generated by python-semantic-release + +### Documentation + +- Add badges (codecov and pypi) to readme + ([#75](https://github.com/rungalileo/galileo-python/pull/75), + [`9f268fe`](https://github.com/rungalileo/galileo-python/commit/9f268fe70b5e4fd7f66732db9ed9636711d7f9d4)) + +- Remove extra brackets ([#76](https://github.com/rungalileo/galileo-python/pull/76), + [`9f0ccc0`](https://github.com/rungalileo/galileo-python/commit/9f0ccc0a89db7c7169e31fa8cb227b1c09095ebf)) + +### Testing + +- Add unit tests for experiments ([#71](https://github.com/rungalileo/galileo-python/pull/71), + [`19870ee`](https://github.com/rungalileo/galileo-python/commit/19870ee044a2a76c0dffa5cfc1082f629b832643)) + + +## v0.2.3 (2025-03-21) + +### Bug Fixes + +- Move from packaging.version import Version to openai section + ([#74](https://github.com/rungalileo/galileo-python/pull/74), + [`085e989`](https://github.com/rungalileo/galileo-python/commit/085e989e2f548ee016a1c6b153b8c5b429364a13)) + +### Chores + +- **release**: V0.2.3 + ([`5aa9b41`](https://github.com/rungalileo/galileo-python/commit/5aa9b41dbf7e0fcd7db492d095702e23780c611a)) + +Automatically generated by python-semantic-release + + +## v0.2.2 (2025-03-18) + +### Bug Fixes + +- Run_experiment() ([#73](https://github.com/rungalileo/galileo-python/pull/73), + [`209ef0f`](https://github.com/rungalileo/galileo-python/commit/209ef0f0c696c3c24a8d2bdbae3297f62984c16e)) + +### Chores + +- **release**: V0.2.2 + ([`2a00be2`](https://github.com/rungalileo/galileo-python/commit/2a00be2635efb6206f861925438c16a7d9569df0)) + +Automatically generated by python-semantic-release + + +## v0.2.1 (2025-03-18) + +### Bug Fixes + +- Fixup after refactoring: PromptTemplates + ([#72](https://github.com/rungalileo/galileo-python/pull/72), + [`cae8e70`](https://github.com/rungalileo/galileo-python/commit/cae8e7097425ef72b6f137e7b5aa0fccbd28dde1)) + +### Chores + +- Add more informative console messages after running an experiment + ([#70](https://github.com/rungalileo/galileo-python/pull/70), + [`fff4d0c`](https://github.com/rungalileo/galileo-python/commit/fff4d0c28af0e0d0ca593485d67a2d89ed21defe)) + +- **release**: V0.2.1 + ([`780b1b9`](https://github.com/rungalileo/galileo-python/commit/780b1b98ac902bdca96ed85fa6fd055500b0d8e5)) + +Automatically generated by python-semantic-release + + +## v0.2.0 (2025-03-18) + +### Bug Fixes + +- Langgraph metadata errors in the Langchain handler + ([#62](https://github.com/rungalileo/galileo-python/pull/62), + [`a8f8133`](https://github.com/rungalileo/galileo-python/commit/a8f8133c196c2c8ac233574fc87fcab520a76f99)) + +Co-authored-by: Andrii Soldatenko + +- Log tools on LLM span in OpenAI decorator + ([#47](https://github.com/rungalileo/galileo-python/pull/47), + [`d2ef9aa`](https://github.com/rungalileo/galileo-python/commit/d2ef9aae94ecf9a68d268eaad7483512fe541026)) + +Co-authored-by: ajaynayak + +- Making sure the final span output bubbles up to the trace during flush + ([#64](https://github.com/rungalileo/galileo-python/pull/64), + [`bb70952`](https://github.com/rungalileo/galileo-python/commit/bb7095265b53b9c3c5fe121c80a83fcd4430b1b3)) + +Co-authored-by: Andrii Soldatenko + +### Chores + +- **release**: V0.2.0 + ([`7d39fcb`](https://github.com/rungalileo/galileo-python/commit/7d39fcb17f7bebc4fde0ff6f9ee1b8a20b642f42)) + +Automatically generated by python-semantic-release + +### Features + +- Add prompt_settings to run_experiment + ([#67](https://github.com/rungalileo/galileo-python/pull/67), + [`02de6ea`](https://github.com/rungalileo/galileo-python/commit/02de6eabec3c83288fcdbcc78c64388a5c2ef528)) + +- Adding a test and updating the sdk version + ([#69](https://github.com/rungalileo/galileo-python/pull/69), + [`180e8b4`](https://github.com/rungalileo/galileo-python/commit/180e8b4c26510caa7e8ec7d23a862b8f3b643fe8)) + +- Implement properly run_experiments with datasets + ([#66](https://github.com/rungalileo/galileo-python/pull/66), + [`17879ef`](https://github.com/rungalileo/galileo-python/commit/17879efd75f7d0207d798476ef58d8a579ea9ff1)) + +- Use @log with ThreadPoolExecutor ([#65](https://github.com/rungalileo/galileo-python/pull/65), + [`f13c746`](https://github.com/rungalileo/galileo-python/commit/f13c746ca253fed21c4de9eaa5e512c7c9427e58)) + +### Refactoring + +- Newly added classes to align with our naming convention + ([#63](https://github.com/rungalileo/galileo-python/pull/63), + [`c540c2a`](https://github.com/rungalileo/galileo-python/commit/c540c2a8e6f3e99ea75314cb443ff9a38d632971)) + +Co-authored-by: Andrii Soldatenko + + +## v0.1.0 (2025-03-14) + +### Bug Fixes + +- Adding init, reset, and flush_all methods to galileo_context, adding tests, fixing existing tests + ([#28](https://github.com/rungalileo/galileo-python/pull/28), + [`569d1d0`](https://github.com/rungalileo/galileo-python/commit/569d1d07686f54b434081f1f79fe5c318e8873f9)) + +- Change job name according to new api version + ([#52](https://github.com/rungalileo/galileo-python/pull/52), + [`9c9352d`](https://github.com/rungalileo/galileo-python/commit/9c9352dc51f8591a33cf411d580707bfc18162d8)) + +- Fixing an issue with parsing OpenAI tool calls outputs + ([#36](https://github.com/rungalileo/galileo-python/pull/36), + [`55c0fe4`](https://github.com/rungalileo/galileo-python/commit/55c0fe44445b6a3b3858978853dd48057bc0452f)) + +- Fixing get log_stream by name ([#40](https://github.com/rungalileo/galileo-python/pull/40), + [`2571024`](https://github.com/rungalileo/galileo-python/commit/257102433d2ab765a11526481d6b2f7c729b6b61)) + +- Output parsing for retriever spans ([#34](https://github.com/rungalileo/galileo-python/pull/34), + [`c87f926`](https://github.com/rungalileo/galileo-python/commit/c87f9268b8ef681ed8535ec0aad5e5b612a71243)) + +- Serialization of non-serialized types or classes + ([#48](https://github.com/rungalileo/galileo-python/pull/48), + [`39d7611`](https://github.com/rungalileo/galileo-python/commit/39d761164b4659f4313cbe05bcf0246371520bbe)) + +- Serializing trace, workflow, and tool span inputs and outputs + ([#41](https://github.com/rungalileo/galileo-python/pull/41), + [`e54cb95`](https://github.com/rungalileo/galileo-python/commit/e54cb9585ffbf6cdbb9e553f76c1fc836b37e36b)) + +- Set min Python version for ruff to py39 + ([#23](https://github.com/rungalileo/galileo-python/pull/23), + [`fcfc0d1`](https://github.com/rungalileo/galileo-python/commit/fcfc0d11937bcde62f38d0e21c2319afa9af3d8f)) + +- Typo inside ./scripts/auto-generate-api-client.sh + ([#32](https://github.com/rungalileo/galileo-python/pull/32), + [`252c061`](https://github.com/rungalileo/galileo-python/commit/252c061d7798cef822b24dc6819f49e5864a7bf9)) + +- Update add_llm_span example ([#39](https://github.com/rungalileo/galileo-python/pull/39), + [`e01838f`](https://github.com/rungalileo/galileo-python/commit/e01838ffe95181133c455f21bc57cc5f1aacd297)) + +### Chores + +- Add missing keys to `pyproject.toml` ([#57](https://github.com/rungalileo/galileo-python/pull/57), + [`500ac21`](https://github.com/rungalileo/galileo-python/commit/500ac21f6201c738a59546b8d602ca947575992f)) + +- Fix path to `__init__` ([#61](https://github.com/rungalileo/galileo-python/pull/61), + [`8d6425d`](https://github.com/rungalileo/galileo-python/commit/8d6425df53510f67f543192e014f694a894651f3)) + +- Remove js dir and files ([#56](https://github.com/rungalileo/galileo-python/pull/56), + [`962ae5d`](https://github.com/rungalileo/galileo-python/commit/962ae5da831da3d10f5748aa1333ef6e6fe1b6fe)) + +- Set version field in `pyproject.toml` correctly + ([#60](https://github.com/rungalileo/galileo-python/pull/60), + [`1560b76`](https://github.com/rungalileo/galileo-python/commit/1560b7615df8bf0320ab531bcd3f87c5fa41495d)) + +- Setup repo + package similarly to our other Python repos + ([#25](https://github.com/rungalileo/galileo-python/pull/25), + [`4daac02`](https://github.com/rungalileo/galileo-python/commit/4daac020776875d2d1e174b300f558f68448671e)) + +- **deps**: Bump `galileo-core` to v3.2+ + ([#29](https://github.com/rungalileo/galileo-python/pull/29), + [`ed234e3`](https://github.com/rungalileo/galileo-python/commit/ed234e3495281409095db3243cff8b01143cd41d)) + +- **deps**: Bump codecov/codecov-action from 5.3.1 to 5.4.0 + ([#42](https://github.com/rungalileo/galileo-python/pull/42), + [`41d7955`](https://github.com/rungalileo/galileo-python/commit/41d79556106db802b2013b40bbf7ac2a65b5899f)) + +- **deps**: Bump python-semantic-release/python-semantic-release from 9.20.0 to 9.21.0 + ([#43](https://github.com/rungalileo/galileo-python/pull/43), + [`95543ce`](https://github.com/rungalileo/galileo-python/commit/95543cec5afb5df96b75439ab7888065f805c4e8)) + +- **release**: V0.1.0 + ([`e443300`](https://github.com/rungalileo/galileo-python/commit/e4433000f2be7a6c2a707a6d369f2f55d5028bb1)) + +Automatically generated by python-semantic-release + +### Continuous Integration + +- Bump python-semantic-release/python-semantic-release from 9.17.0 to 9.20.0 + ([#26](https://github.com/rungalileo/galileo-python/pull/26), + [`ac9aaab`](https://github.com/rungalileo/galileo-python/commit/ac9aaab3539e1ff3e1603ea8371064e6913ddf36)) + +### Documentation + +- Add reference docs to the more client-facing pages + ([#37](https://github.com/rungalileo/galileo-python/pull/37), + [`08b3457`](https://github.com/rungalileo/galileo-python/commit/08b3457d9957523d77e480b4c2dce9e63bb0f5f6)) + +Co-authored-by: ajaynayak + +- Add small note about poetry shell ([#12](https://github.com/rungalileo/galileo-python/pull/12), + [`d598aae`](https://github.com/rungalileo/galileo-python/commit/d598aae0fb61c700e9ea56fdeb72154aeb343aea)) + +### Features + +- Add streaming support to openai wrapper + ([#31](https://github.com/rungalileo/galileo-python/pull/31), + [`78687c7`](https://github.com/rungalileo/galileo-python/commit/78687c72cbc4ac10df5630209ffe9b09a4dd56f2)) + +- Adding a client-type header to all requests + ([#54](https://github.com/rungalileo/galileo-python/pull/54), + [`764ee2d`](https://github.com/rungalileo/galileo-python/commit/764ee2d00cb3493304c2d1af51ce1f1b97a5d6e1)) + +- Adding a way to conclude all spans in a trace; restoring defaults in the decorator + ([#35](https://github.com/rungalileo/galileo-python/pull/35), + [`130fe02`](https://github.com/rungalileo/galileo-python/commit/130fe0218f2df201b86366959b165034a44c74e9)) + +- Catch and handle errors throughout the client + ([#46](https://github.com/rungalileo/galileo-python/pull/46), + [`d6a82ee`](https://github.com/rungalileo/galileo-python/commit/d6a82eed09b1923d8d25c792e804857419b62a98)) + +- Changes to support the new core logging schemas + ([#30](https://github.com/rungalileo/galileo-python/pull/30), + [`a2c6ca8`](https://github.com/rungalileo/galileo-python/commit/a2c6ca85efb7ad1d5e1e581219c88c91484fd3b6)) + +Changes to the client based on the following core and api changes: rungalileo/core#232 + rungalileo/api#3489 + +There were some DX changes I made in this PR that will need to get moved to core: - renaming + user_metadata to metadata for the logging functions - allowing more flexible types to be used for + documents in the add_retriever_span() method. The current traces_logger method is too restrictive + (forces a user to specify a list of dicts or a list of Documents, else throws an error). Since + we're using function decorators, we need to be more permissive of method outputs which will map to + the retriever documents field. + +- Decorator should create trace but reraise original exception + ([#45](https://github.com/rungalileo/galileo-python/pull/45), + [`d3dbb7a`](https://github.com/rungalileo/galileo-python/commit/d3dbb7a0cb2a6e37c0684ec18cd55ee1113e173f)) + +- Implement get/create for prompt templates + ([#49](https://github.com/rungalileo/galileo-python/pull/49), + [`c647da3`](https://github.com/rungalileo/galileo-python/commit/c647da3bd1ac44033b493d7ca7f8ef70877a7e68)) + +- Langchain callback ([#44](https://github.com/rungalileo/galileo-python/pull/44), + [`b6dd9a3`](https://github.com/rungalileo/galileo-python/commit/b6dd9a3514b3dba7e2a48beff42342b352b2e60e)) + +- Replace app.galileo.ai with api.galileo.ai if users specify it incorrectly + ([#53](https://github.com/rungalileo/galileo-python/pull/53), + [`9a7ac9e`](https://github.com/rungalileo/galileo-python/commit/9a7ac9e623808341c9bf72ffc58543943bf6d27f)) + +- Run experiment with a runner function and hosted metrics + ([#58](https://github.com/rungalileo/galileo-python/pull/58), + [`5c37ff7`](https://github.com/rungalileo/galileo-python/commit/5c37ff7516d020a8195c23fe30907e9839f2d9fe)) + +Co-authored-by: Andrii Soldatenko + +Co-authored-by: ajaynayak + +- Run experiment with run_prompt and hosted metrics + ([#50](https://github.com/rungalileo/galileo-python/pull/50), + [`c78908f`](https://github.com/rungalileo/galileo-python/commit/c78908f699006311a1249c0341ed9f34f589dcde)) + +- Updating the readme and pyproject for release + ([#59](https://github.com/rungalileo/galileo-python/pull/59), + [`867a327`](https://github.com/rungalileo/galileo-python/commit/867a327e92395451feb3e2a76b301a10a3cb8f99)) + +### Refactoring + +- Removing `project` and `log_stream` from the `log` decorator + ([#55](https://github.com/rungalileo/galileo-python/pull/55), + [`d348dbd`](https://github.com/rungalileo/galileo-python/commit/d348dbdc1dd23595c8bfbaee736112681e11ea21)) + +### Testing + +- Add tests to emulate openai errors and galileo api errors + ([#38](https://github.com/rungalileo/galileo-python/pull/38), + [`16f73bb`](https://github.com/rungalileo/galileo-python/commit/16f73bb6f63a755a8d4ee3a907b4d938a22519f9)) + +- Adding unit tests for openai wrapper ([#27](https://github.com/rungalileo/galileo-python/pull/27), + [`fdc15d1`](https://github.com/rungalileo/galileo-python/commit/fdc15d144b1e10199d64d3d5bc706f7da80e2a94)) + +- Dont ignore async test ([#33](https://github.com/rungalileo/galileo-python/pull/33), + [`6f75fa8`](https://github.com/rungalileo/galileo-python/commit/6f75fa8e134cc1e8ec82e2856680e354386b5414)) diff --git a/src/splunk_ao/experiment.py b/src/splunk_ao/experiment.py index 6cd4bc8..4ea2cfe 100644 --- a/src/splunk_ao/experiment.py +++ b/src/splunk_ao/experiment.py @@ -3,7 +3,6 @@ import builtins import datetime import re -import warnings from collections.abc import Iterator from time import sleep from typing import TYPE_CHECKING, Any @@ -1126,7 +1125,6 @@ def monitor_progress( poll_interval_seconds: float = 2.0, *, timeout_seconds: float | None = 3600.0, - job_id: str | None = None, ) -> None: """ Monitor the progress of the experiment with a progress bar. @@ -1141,9 +1139,6 @@ def monitor_progress( timeout_seconds : float or None, optional Maximum seconds to wait before raising TimeoutError. Defaults to 3600.0 (one hour). Pass None to wait indefinitely (not recommended). - job_id : str or None, optional - Deprecated. This parameter is ignored; it existed in a prior version - that polled the jobs table, which has been retired. Returns ------- @@ -1168,14 +1163,6 @@ def monitor_progress( experiment.monitor_progress() """ - if job_id is not None: - warnings.warn( - "The 'job_id' parameter of monitor_progress() is deprecated and will be removed in a future release. " - "Progress is now tracked directly via experiment status; the job_id value is ignored.", - DeprecationWarning, - stacklevel=2, - ) - if self.id is None: raise ValueError("Experiment ID is not set. Cannot monitor progress for a local-only experiment.") if self.project_id is None: diff --git a/tests/test_experiment_progress.py b/tests/test_experiment_progress.py index 8f3d016..525dfbe 100644 --- a/tests/test_experiment_progress.py +++ b/tests/test_experiment_progress.py @@ -89,17 +89,6 @@ def test_raises_without_project_id(self): with pytest.raises(ValueError, match="Project ID is not set"): exp.monitor_progress() - @patch("splunk_ao.experiment.Experiment.get_status") - @patch("splunk_ao.experiment.sleep", return_value=None) - def test_deprecated_job_id_warns(self, mock_sleep, mock_get_status): - # Given: an experiment that is already complete, and a caller passing the deprecated job_id - mock_get_status.return_value = _make_status(100.0) - exp = _make_experiment() - - # When/Then: monitor_progress emits a DeprecationWarning when job_id is supplied - with pytest.warns(DeprecationWarning, match="job_id"): - exp.monitor_progress(job_id="some-old-job-id") - @patch("splunk_ao.experiment.Experiment.get_status") @patch("splunk_ao.experiment.sleep", return_value=None) def test_raises_runtime_error_on_failed_experiment(self, mock_sleep, mock_get_status):