refactor(generative_ai/evaluation): migrate vertexai.generative_models to genai SDK

generative_ai/evaluation/get_rouge_score.py

@@ -11,19 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+# [START generativeaionvertexai_evaluation_get_rouge_score]
+
 import os
 
+import pandas as pd
+
+import vertexai
 from vertexai.preview.evaluation import EvalResult
+from vertexai.preview.evaluation import EvalTask
 
 PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
 
 
 def get_rouge_score() -> EvalResult:
-    # [START generativeaionvertexai_evaluation_get_rouge_score]
-    import pandas as pd
-
-    import vertexai
-    from vertexai.preview.evaluation import EvalTask
 
     # TODO(developer): Update & uncomment line below
     # PROJECT_ID = "your-project-id"

generative_ai/evaluation/noxfile_config.py

@@ -22,7 +22,7 @@
 
 TEST_CONFIG_OVERRIDE = {
     # You can opt out from the test for specific Python versions.
-    "ignored_versions": ["3.8", "3.9", "3.11", "3.12", "3.13"],
+    "ignored_versions": ["3.8", "3.9", "3.10", "3.12", "3.13"],
     # Old samples are opted out of enforcing Python type hints
     # All new samples should feature them
     "enforce_type_hints": True,

generative_ai/evaluation/pairwise_summarization_quality.py

@@ -11,28 +11,50 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+# [START generativeaionvertexai_evaluation_pairwise_summarization_quality]
+
 import os
 
-from vertexai.preview.evaluation import EvalResult
+import pandas as pd
 
+from google import genai
+from google.genai import types
+
+import vertexai
+from vertexai.preview.evaluation import EvalResult
+from vertexai.evaluation import (
+    EvalTask,
+    PairwiseMetric,
+    MetricPromptTemplateExamples,
+)
+
+# TODO (developer) set GOOGLE_CLOUD_PROJECT and REGION_ID
+# environment variables before running.
 PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
+LOCATION = os.getenv("REGION_ID")
+BASELINE_MODEL = "gemini-2.5-flash"
 
 
-def evaluate_output() -> EvalResult:
-    # [START generativeaionvertexai_evaluation_pairwise_summarization_quality]
-    import pandas as pd
-
-    import vertexai
-    from vertexai.generative_models import GenerativeModel
-    from vertexai.evaluation import (
-        EvalTask,
-        PairwiseMetric,
-        MetricPromptTemplateExamples,
+def custom_model_fn(prompt: str) -> str:
+    """Generates text from a prompt using the baseline Gemini model via Vertex AI."""
+
+    genai_client = genai.Client(vertexai=True, project=PROJECT_ID, location=LOCATION)
+
+    genai_config = types.GenerateContentConfig(temperature=0.4)
+
+    response = genai_client.models.generate_content(
+        model=BASELINE_MODEL, contents=prompt, config=genai_config
     )
 
-    # TODO(developer): Update & uncomment line below
-    # PROJECT_ID = "your-project-id"
-    vertexai.init(project=PROJECT_ID, location="us-central1")
+    return response.text
+
+
+def evaluate_output() -> EvalResult:
+    """
+    Evaluates a candidate model's summarization quality
+    against a baseline model using Vertex AI.
+    """
 
     prompt = """
     Summarize the text such that a five-year-old can understand.
@@ -49,15 +71,9 @@ def evaluate_output() -> EvalResult:
     efficient, environmentally conscious urban transportation.
     """
 
-    eval_dataset = pd.DataFrame({"prompt": [prompt]})
-
-    # Baseline model for pairwise comparison
-    baseline_model = GenerativeModel("gemini-2.0-flash-lite-001")
+    vertexai.init(project=PROJECT_ID, location=LOCATION)
 
-    # Candidate model for pairwise comparison
-    candidate_model = GenerativeModel(
-        "gemini-2.0-flash-001", generation_config={"temperature": 0.4}
-    )
+    eval_dataset = pd.DataFrame({"prompt": [prompt]})
 
     prompt_template = MetricPromptTemplateExamples.get_prompt_template(
         "pairwise_summarization_quality"
@@ -66,21 +82,25 @@ def evaluate_output() -> EvalResult:
     summarization_quality_metric = PairwiseMetric(
         metric="pairwise_summarization_quality",
         metric_prompt_template=prompt_template,
-        baseline_model=baseline_model,
+        baseline_model=BASELINE_MODEL,
     )
 
     eval_task = EvalTask(
         dataset=eval_dataset,
         metrics=[summarization_quality_metric],
         experiment="pairwise-experiment",
     )
-    result = eval_task.evaluate(model=candidate_model)
+    result = eval_task.evaluate(
+        model=custom_model_fn, experiment_run_name="genai-client-vs-legacy-baseline-6"
+    )
 
     baseline_model_response = result.metrics_table["baseline_model_response"].iloc[0]
     candidate_model_response = result.metrics_table["response"].iloc[0]
+
     winner_model = result.metrics_table[
         "pairwise_summarization_quality/pairwise_choice"
     ].iloc[0]
+
     explanation = result.metrics_table[
         "pairwise_summarization_quality/explanation"
     ].iloc[0]

generative_ai/evaluation/requirements-test.txt

@@ -1,4 +1,4 @@
 backoff==2.2.1
-google-api-core==2.19.0
-pytest==9.0.3; python_version >= "3.10"
-pytest-asyncio==0.23.6
+google-api-core==2.31.0
+pytest==9.0.3
+pytest-asyncio==1.4.0

generative_ai/evaluation/requirements.txt

@@ -1,13 +1,4 @@
-pandas==2.2.3; python_version == '3.7'
-pandas==2.2.3; python_version == '3.8'
-pandas==2.2.3; python_version > '3.8'
-pillow==12.2.0
-google-cloud-aiplatform[full]==1.157.0
-sentencepiece==0.2.1
-google-auth==2.38.0
-anthropic[vertex]==0.28.0
-langchain-core==1.4.0
-langchain-google-vertexai==3.2.3
-numpy<3
-openai==1.68.2
-immutabledict==4.2.0
+pandas==2.3.3
+google-auth==2.55.2
+google-cloud-aiplatform[full]==1.161.0
+google-genai==2.12.1