feat: add evaluation metrics and autorater configuration to the AI Platform v1 API

google/cloud/aiplatform_v1/__init__.py

@@ -261,6 +261,9 @@
 from .types.evaluated_annotation import ErrorAnalysisAnnotation
 from .types.evaluated_annotation import EvaluatedAnnotation
 from .types.evaluated_annotation import EvaluatedAnnotationExplanation
+from .types.evaluation_service import AggregationOutput
+from .types.evaluation_service import AggregationResult
+from .types.evaluation_service import AutoraterConfig
 from .types.evaluation_service import BleuInput
 from .types.evaluation_service import BleuInstance
 from .types.evaluation_service import BleuMetricValue
@@ -274,8 +277,14 @@
 from .types.evaluation_service import CometInstance
 from .types.evaluation_service import CometResult
 from .types.evaluation_service import CometSpec
+from .types.evaluation_service import ComputationBasedMetricSpec
+from .types.evaluation_service import ContentMap
+from .types.evaluation_service import CustomOutput
+from .types.evaluation_service import CustomOutputFormatConfig
+from .types.evaluation_service import EvaluateDatasetResponse
 from .types.evaluation_service import EvaluateInstancesRequest
 from .types.evaluation_service import EvaluateInstancesResponse
+from .types.evaluation_service import EvaluationDataset
 from .types.evaluation_service import ExactMatchInput
 from .types.evaluation_service import ExactMatchInstance
 from .types.evaluation_service import ExactMatchMetricValue
@@ -293,10 +302,15 @@
 from .types.evaluation_service import GroundednessInstance
 from .types.evaluation_service import GroundednessResult
 from .types.evaluation_service import GroundednessSpec
+from .types.evaluation_service import LLMBasedMetricSpec
+from .types.evaluation_service import Metric
+from .types.evaluation_service import MetricResult
 from .types.evaluation_service import MetricxInput
 from .types.evaluation_service import MetricxInstance
 from .types.evaluation_service import MetricxResult
 from .types.evaluation_service import MetricxSpec
+from .types.evaluation_service import OutputConfig
+from .types.evaluation_service import OutputInfo
 from .types.evaluation_service import PairwiseMetricInput
 from .types.evaluation_service import PairwiseMetricInstance
 from .types.evaluation_service import PairwiseMetricResult
@@ -313,6 +327,7 @@
 from .types.evaluation_service import PointwiseMetricInstance
 from .types.evaluation_service import PointwiseMetricResult
 from .types.evaluation_service import PointwiseMetricSpec
+from .types.evaluation_service import PredefinedMetricSpec
 from .types.evaluation_service import QuestionAnsweringCorrectnessInput
 from .types.evaluation_service import QuestionAnsweringCorrectnessInstance
 from .types.evaluation_service import QuestionAnsweringCorrectnessResult
@@ -329,6 +344,7 @@
 from .types.evaluation_service import QuestionAnsweringRelevanceInstance
 from .types.evaluation_service import QuestionAnsweringRelevanceResult
 from .types.evaluation_service import QuestionAnsweringRelevanceSpec
+from .types.evaluation_service import RawOutput
 from .types.evaluation_service import RougeInput
 from .types.evaluation_service import RougeInstance
 from .types.evaluation_service import RougeMetricValue
@@ -1023,6 +1039,8 @@
 from .types.training_pipeline import StratifiedSplit
 from .types.training_pipeline import TimestampSplit
 from .types.training_pipeline import TrainingPipeline
+from .types.tuning_job import EvaluateDatasetRun
+from .types.tuning_job import EvaluationConfig
 from .types.tuning_job import PreTunedModel
 from .types.tuning_job import SupervisedHyperParameters
 from .types.tuning_job import SupervisedTuningDatasetDistribution
@@ -1253,6 +1271,8 @@ def _get_version(dependency_name):
     "AddExecutionEventsRequest",
     "AddExecutionEventsResponse",
     "AddTrialMeasurementRequest",
+    "AggregationOutput",
+    "AggregationResult",
     "Annotation",
     "AnnotationSpec",
     "ApiAuth",
@@ -1270,6 +1290,7 @@ def _get_version(dependency_name):
     "AugmentPromptRequest",
     "AugmentPromptResponse",
     "AutomaticResources",
+    "AutoraterConfig",
     "AutoscalingMetricSpec",
     "AvroSource",
     "BatchCancelPipelineJobsOperationMetadata",
@@ -1338,11 +1359,13 @@ def _get_version(dependency_name):
     "CometSpec",
     "CompleteTrialRequest",
     "CompletionStats",
+    "ComputationBasedMetricSpec",
     "ComputeTokensRequest",
     "ComputeTokensResponse",
     "ContainerRegistryDestination",
     "ContainerSpec",
     "Content",
+    "ContentMap",
     "Context",
     "CopyModelOperationMetadata",
     "CopyModelRequest",
@@ -1419,6 +1442,8 @@ def _get_version(dependency_name):
     "CsvSource",
     "CustomJob",
     "CustomJobSpec",
+    "CustomOutput",
+    "CustomOutputFormatConfig",
     "DataFoundryServiceClient",
     "DataItem",
     "DataItemView",
@@ -1513,10 +1538,14 @@ def _get_version(dependency_name):
     "EntityType",
     "EnvVar",
     "ErrorAnalysisAnnotation",
+    "EvaluateDatasetResponse",
+    "EvaluateDatasetRun",
     "EvaluateInstancesRequest",
     "EvaluateInstancesResponse",
     "EvaluatedAnnotation",
     "EvaluatedAnnotationExplanation",
+    "EvaluationConfig",
+    "EvaluationDataset",
     "EvaluationServiceClient",
     "Event",
     "EventActions",
@@ -1704,6 +1733,7 @@ def _get_version(dependency_name):
     "JiraSource",
     "JobServiceClient",
     "JobState",
+    "LLMBasedMetricSpec",
     "LargeModelReference",
     "LineageSubgraph",
     "ListAnnotationsRequest",
@@ -1828,6 +1858,8 @@ def _get_version(dependency_name):
     "MetadataSchema",
     "MetadataServiceClient",
     "MetadataStore",
+    "Metric",
+    "MetricResult",
     "MetricxInput",
     "MetricxInstance",
     "MetricxResult",
@@ -1885,7 +1917,9 @@ def _get_version(dependency_name):
     "NotebookRuntimeType",
     "NotebookServiceClient",
     "NotebookSoftwareConfig",
+    "OutputConfig",
     "OutputFieldSpec",
+    "OutputInfo",
     "PSCAutomationConfig",
     "PSCAutomationState",
     "PairwiseChoice",
@@ -1924,6 +1958,7 @@ def _get_version(dependency_name):
     "PostStartupScriptConfig",
     "PreTunedModel",
     "PrebuiltVoiceConfig",
+    "PredefinedMetricSpec",
     "PredefinedSplit",
     "PredictRequest",
     "PredictRequestResponseLoggingConfig",
@@ -1984,6 +2019,7 @@ def _get_version(dependency_name):
     "RagQuery",
     "RagRetrievalConfig",
     "RagVectorDbConfig",
+    "RawOutput",
     "RawPredictRequest",
     "RayLogsSpec",
     "RayMetricSpec",

google/cloud/aiplatform_v1/services/migration_service/client.py

@@ -293,40 +293,40 @@ def parse_dataset_path(path: str) -> Dict[str, str]:
     @staticmethod
     def dataset_path(
         project: str,
+        location: str,
         dataset: str,
     ) -> str:
         """Returns a fully-qualified dataset string."""
-        return "projects/{project}/datasets/{dataset}".format(
+        return "projects/{project}/locations/{location}/datasets/{dataset}".format(
             project=project,
+            location=location,
             dataset=dataset,
         )
 
     @staticmethod
     def parse_dataset_path(path: str) -> Dict[str, str]:
         """Parses a dataset path into its component segments."""
-        m = re.match(r"^projects/(?P<project>.+?)/datasets/(?P<dataset>.+?)$", path)
+        m = re.match(
+            r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/datasets/(?P<dataset>.+?)$",
+            path,
+        )
         return m.groupdict() if m else {}
 
     @staticmethod
     def dataset_path(
         project: str,
-        location: str,
         dataset: str,
     ) -> str:
         """Returns a fully-qualified dataset string."""
-        return "projects/{project}/locations/{location}/datasets/{dataset}".format(
+        return "projects/{project}/datasets/{dataset}".format(
             project=project,
-            location=location,
             dataset=dataset,
         )
 
     @staticmethod
     def parse_dataset_path(path: str) -> Dict[str, str]:
         """Parses a dataset path into its component segments."""
-        m = re.match(
-            r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/datasets/(?P<dataset>.+?)$",
-            path,
-        )
+        m = re.match(r"^projects/(?P<project>.+?)/datasets/(?P<dataset>.+?)$", path)
         return m.groupdict() if m else {}
 
     @staticmethod

google/cloud/aiplatform_v1/types/__init__.py

@@ -214,6 +214,9 @@
     EvaluatedAnnotationExplanation,
 )
 from .evaluation_service import (
+    AggregationOutput,
+    AggregationResult,
+    AutoraterConfig,
     BleuInput,
     BleuInstance,
     BleuMetricValue,
@@ -227,8 +230,14 @@
     CometInstance,
     CometResult,
     CometSpec,
+    ComputationBasedMetricSpec,
+    ContentMap,
+    CustomOutput,
+    CustomOutputFormatConfig,
+    EvaluateDatasetResponse,
     EvaluateInstancesRequest,
     EvaluateInstancesResponse,
+    EvaluationDataset,
     ExactMatchInput,
     ExactMatchInstance,
     ExactMatchMetricValue,
@@ -246,10 +255,15 @@
     GroundednessInstance,
     GroundednessResult,
     GroundednessSpec,
+    LLMBasedMetricSpec,
+    Metric,
+    MetricResult,
     MetricxInput,
     MetricxInstance,
     MetricxResult,
     MetricxSpec,
+    OutputConfig,
+    OutputInfo,
     PairwiseMetricInput,
     PairwiseMetricInstance,
     PairwiseMetricResult,
@@ -266,6 +280,7 @@
     PointwiseMetricInstance,
     PointwiseMetricResult,
     PointwiseMetricSpec,
+    PredefinedMetricSpec,
     QuestionAnsweringCorrectnessInput,
     QuestionAnsweringCorrectnessInstance,
     QuestionAnsweringCorrectnessResult,
@@ -282,6 +297,7 @@
     QuestionAnsweringRelevanceInstance,
     QuestionAnsweringRelevanceResult,
     QuestionAnsweringRelevanceSpec,
+    RawOutput,
     RougeInput,
     RougeInstance,
     RougeMetricValue,
@@ -1136,6 +1152,8 @@
     TrainingPipeline,
 )
 from .tuning_job import (
+    EvaluateDatasetRun,
+    EvaluationConfig,
     PreTunedModel,
     SupervisedHyperParameters,
     SupervisedTuningDatasetDistribution,
@@ -1392,6 +1410,9 @@
     "ErrorAnalysisAnnotation",
     "EvaluatedAnnotation",
     "EvaluatedAnnotationExplanation",
+    "AggregationOutput",
+    "AggregationResult",
+    "AutoraterConfig",
     "BleuInput",
     "BleuInstance",
     "BleuMetricValue",
@@ -1405,8 +1426,14 @@
     "CometInstance",
     "CometResult",
     "CometSpec",
+    "ComputationBasedMetricSpec",
+    "ContentMap",
+    "CustomOutput",
+    "CustomOutputFormatConfig",
+    "EvaluateDatasetResponse",
     "EvaluateInstancesRequest",
     "EvaluateInstancesResponse",
+    "EvaluationDataset",
     "ExactMatchInput",
     "ExactMatchInstance",
     "ExactMatchMetricValue",
@@ -1424,10 +1451,15 @@
     "GroundednessInstance",
     "GroundednessResult",
     "GroundednessSpec",
+    "LLMBasedMetricSpec",
+    "Metric",
+    "MetricResult",
     "MetricxInput",
     "MetricxInstance",
     "MetricxResult",
     "MetricxSpec",
+    "OutputConfig",
+    "OutputInfo",
     "PairwiseMetricInput",
     "PairwiseMetricInstance",
     "PairwiseMetricResult",
@@ -1444,6 +1476,7 @@
     "PointwiseMetricInstance",
     "PointwiseMetricResult",
     "PointwiseMetricSpec",
+    "PredefinedMetricSpec",
     "QuestionAnsweringCorrectnessInput",
     "QuestionAnsweringCorrectnessInstance",
     "QuestionAnsweringCorrectnessResult",
@@ -1460,6 +1493,7 @@
     "QuestionAnsweringRelevanceInstance",
     "QuestionAnsweringRelevanceResult",
     "QuestionAnsweringRelevanceSpec",
+    "RawOutput",
     "RougeInput",
     "RougeInstance",
     "RougeMetricValue",
@@ -2142,6 +2176,8 @@
     "StratifiedSplit",
     "TimestampSplit",
     "TrainingPipeline",
+    "EvaluateDatasetRun",
+    "EvaluationConfig",
     "PreTunedModel",
     "SupervisedHyperParameters",
     "SupervisedTuningDatasetDistribution",