From aa4ace614e92fea581f55e1730ace9dcafb10fac Mon Sep 17 00:00:00 2001 From: David del Real Sifuentes Date: Thu, 16 Jul 2026 22:40:56 +0000 Subject: [PATCH] refactor(generative_ai/evaluation): migrate vertexai.generative_models to genai SDK - Migrated evaluation samples to use the new `google.genai` SDK instead of `vertexai.generative_models`. - Removed unnecessary dependencies from requirements after testing confirmed they are no longer needed. --- generative_ai/evaluation/get_rouge_score.py | 12 ++-- generative_ai/evaluation/noxfile_config.py | 2 +- .../pairwise_summarization_quality.py | 68 ++++++++++++------- .../evaluation/requirements-test.txt | 6 +- generative_ai/evaluation/requirements.txt | 17 ++--- 5 files changed, 59 insertions(+), 46 deletions(-) diff --git a/generative_ai/evaluation/get_rouge_score.py b/generative_ai/evaluation/get_rouge_score.py index 579c0931374..323750f95d6 100644 --- a/generative_ai/evaluation/get_rouge_score.py +++ b/generative_ai/evaluation/get_rouge_score.py @@ -11,19 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# [START generativeaionvertexai_evaluation_get_rouge_score] + import os +import pandas as pd + +import vertexai from vertexai.preview.evaluation import EvalResult +from vertexai.preview.evaluation import EvalTask PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") def get_rouge_score() -> EvalResult: - # [START generativeaionvertexai_evaluation_get_rouge_score] - import pandas as pd - - import vertexai - from vertexai.preview.evaluation import EvalTask # TODO(developer): Update & uncomment line below # PROJECT_ID = "your-project-id" diff --git a/generative_ai/evaluation/noxfile_config.py b/generative_ai/evaluation/noxfile_config.py index 0973c8621c7..1d2019a0b18 100644 --- a/generative_ai/evaluation/noxfile_config.py +++ b/generative_ai/evaluation/noxfile_config.py @@ -22,7 +22,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"], + "ignored_versions": ["3.8", "3.9", "3.10", "3.12", "3.13"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": True, diff --git a/generative_ai/evaluation/pairwise_summarization_quality.py b/generative_ai/evaluation/pairwise_summarization_quality.py index 88c89871904..ea116a5cb8d 100644 --- a/generative_ai/evaluation/pairwise_summarization_quality.py +++ b/generative_ai/evaluation/pairwise_summarization_quality.py @@ -11,28 +11,50 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# [START generativeaionvertexai_evaluation_pairwise_summarization_quality] + import os -from vertexai.preview.evaluation import EvalResult +import pandas as pd +from google import genai +from google.genai import types + +import vertexai +from vertexai.preview.evaluation import EvalResult +from vertexai.evaluation import ( + EvalTask, + PairwiseMetric, + MetricPromptTemplateExamples, +) + +# TODO (developer) set GOOGLE_CLOUD_PROJECT and REGION_ID +# environment variables before running. PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") +LOCATION = os.getenv("REGION_ID") +BASELINE_MODEL = "gemini-2.5-flash" -def evaluate_output() -> EvalResult: - # [START generativeaionvertexai_evaluation_pairwise_summarization_quality] - import pandas as pd - - import vertexai - from vertexai.generative_models import GenerativeModel - from vertexai.evaluation import ( - EvalTask, - PairwiseMetric, - MetricPromptTemplateExamples, +def custom_model_fn(prompt: str) -> str: + """Generates text from a prompt using the baseline Gemini model via Vertex AI.""" + + genai_client = genai.Client(vertexai=True, project=PROJECT_ID, location=LOCATION) + + genai_config = types.GenerateContentConfig(temperature=0.4) + + response = genai_client.models.generate_content( + model=BASELINE_MODEL, contents=prompt, config=genai_config ) - # TODO(developer): Update & uncomment line below - # PROJECT_ID = "your-project-id" - vertexai.init(project=PROJECT_ID, location="us-central1") + return response.text + + +def evaluate_output() -> EvalResult: + """ + Evaluates a candidate model's summarization quality + against a baseline model using Vertex AI. + """ prompt = """ Summarize the text such that a five-year-old can understand. @@ -49,15 +71,9 @@ def evaluate_output() -> EvalResult: efficient, environmentally conscious urban transportation. """ - eval_dataset = pd.DataFrame({"prompt": [prompt]}) - - # Baseline model for pairwise comparison - baseline_model = GenerativeModel("gemini-2.0-flash-lite-001") + vertexai.init(project=PROJECT_ID, location=LOCATION) - # Candidate model for pairwise comparison - candidate_model = GenerativeModel( - "gemini-2.0-flash-001", generation_config={"temperature": 0.4} - ) + eval_dataset = pd.DataFrame({"prompt": [prompt]}) prompt_template = MetricPromptTemplateExamples.get_prompt_template( "pairwise_summarization_quality" @@ -66,7 +82,7 @@ def evaluate_output() -> EvalResult: summarization_quality_metric = PairwiseMetric( metric="pairwise_summarization_quality", metric_prompt_template=prompt_template, - baseline_model=baseline_model, + baseline_model=BASELINE_MODEL, ) eval_task = EvalTask( @@ -74,13 +90,17 @@ def evaluate_output() -> EvalResult: metrics=[summarization_quality_metric], experiment="pairwise-experiment", ) - result = eval_task.evaluate(model=candidate_model) + result = eval_task.evaluate( + model=custom_model_fn, experiment_run_name="genai-client-vs-legacy-baseline-6" + ) baseline_model_response = result.metrics_table["baseline_model_response"].iloc[0] candidate_model_response = result.metrics_table["response"].iloc[0] + winner_model = result.metrics_table[ "pairwise_summarization_quality/pairwise_choice" ].iloc[0] + explanation = result.metrics_table[ "pairwise_summarization_quality/explanation" ].iloc[0] diff --git a/generative_ai/evaluation/requirements-test.txt b/generative_ai/evaluation/requirements-test.txt index baa23bf9c3e..598922c8c25 100644 --- a/generative_ai/evaluation/requirements-test.txt +++ b/generative_ai/evaluation/requirements-test.txt @@ -1,4 +1,4 @@ backoff==2.2.1 -google-api-core==2.19.0 -pytest==9.0.3; python_version >= "3.10" -pytest-asyncio==0.23.6 +google-api-core==2.31.0 +pytest==9.0.3 +pytest-asyncio==1.4.0 diff --git a/generative_ai/evaluation/requirements.txt b/generative_ai/evaluation/requirements.txt index 18ff8773d69..842e7cc88b6 100644 --- a/generative_ai/evaluation/requirements.txt +++ b/generative_ai/evaluation/requirements.txt @@ -1,13 +1,4 @@ -pandas==2.2.3; python_version == '3.7' -pandas==2.2.3; python_version == '3.8' -pandas==2.2.3; python_version > '3.8' -pillow==12.2.0 -google-cloud-aiplatform[full]==1.157.0 -sentencepiece==0.2.1 -google-auth==2.38.0 -anthropic[vertex]==0.28.0 -langchain-core==1.4.0 -langchain-google-vertexai==3.2.3 -numpy<3 -openai==1.68.2 -immutabledict==4.2.0 +pandas==2.3.3 +google-auth==2.55.2 +google-cloud-aiplatform[full]==1.161.0 +google-genai==2.12.1