diff --git a/google/cloud/aiplatform_v1/__init__.py b/google/cloud/aiplatform_v1/__init__.py index 776de81007..fa4243838c 100644 --- a/google/cloud/aiplatform_v1/__init__.py +++ b/google/cloud/aiplatform_v1/__init__.py @@ -261,6 +261,9 @@ from .types.evaluated_annotation import ErrorAnalysisAnnotation from .types.evaluated_annotation import EvaluatedAnnotation from .types.evaluated_annotation import EvaluatedAnnotationExplanation +from .types.evaluation_service import AggregationOutput +from .types.evaluation_service import AggregationResult +from .types.evaluation_service import AutoraterConfig from .types.evaluation_service import BleuInput from .types.evaluation_service import BleuInstance from .types.evaluation_service import BleuMetricValue @@ -274,8 +277,14 @@ from .types.evaluation_service import CometInstance from .types.evaluation_service import CometResult from .types.evaluation_service import CometSpec +from .types.evaluation_service import ComputationBasedMetricSpec +from .types.evaluation_service import ContentMap +from .types.evaluation_service import CustomOutput +from .types.evaluation_service import CustomOutputFormatConfig +from .types.evaluation_service import EvaluateDatasetResponse from .types.evaluation_service import EvaluateInstancesRequest from .types.evaluation_service import EvaluateInstancesResponse +from .types.evaluation_service import EvaluationDataset from .types.evaluation_service import ExactMatchInput from .types.evaluation_service import ExactMatchInstance from .types.evaluation_service import ExactMatchMetricValue @@ -293,10 +302,15 @@ from .types.evaluation_service import GroundednessInstance from .types.evaluation_service import GroundednessResult from .types.evaluation_service import GroundednessSpec +from .types.evaluation_service import LLMBasedMetricSpec +from .types.evaluation_service import Metric +from .types.evaluation_service import MetricResult from .types.evaluation_service import MetricxInput from .types.evaluation_service import MetricxInstance from .types.evaluation_service import MetricxResult from .types.evaluation_service import MetricxSpec +from .types.evaluation_service import OutputConfig +from .types.evaluation_service import OutputInfo from .types.evaluation_service import PairwiseMetricInput from .types.evaluation_service import PairwiseMetricInstance from .types.evaluation_service import PairwiseMetricResult @@ -313,6 +327,7 @@ from .types.evaluation_service import PointwiseMetricInstance from .types.evaluation_service import PointwiseMetricResult from .types.evaluation_service import PointwiseMetricSpec +from .types.evaluation_service import PredefinedMetricSpec from .types.evaluation_service import QuestionAnsweringCorrectnessInput from .types.evaluation_service import QuestionAnsweringCorrectnessInstance from .types.evaluation_service import QuestionAnsweringCorrectnessResult @@ -329,6 +344,7 @@ from .types.evaluation_service import QuestionAnsweringRelevanceInstance from .types.evaluation_service import QuestionAnsweringRelevanceResult from .types.evaluation_service import QuestionAnsweringRelevanceSpec +from .types.evaluation_service import RawOutput from .types.evaluation_service import RougeInput from .types.evaluation_service import RougeInstance from .types.evaluation_service import RougeMetricValue @@ -1023,6 +1039,8 @@ from .types.training_pipeline import StratifiedSplit from .types.training_pipeline import TimestampSplit from .types.training_pipeline import TrainingPipeline +from .types.tuning_job import EvaluateDatasetRun +from .types.tuning_job import EvaluationConfig from .types.tuning_job import PreTunedModel from .types.tuning_job import SupervisedHyperParameters from .types.tuning_job import SupervisedTuningDatasetDistribution @@ -1253,6 +1271,8 @@ def _get_version(dependency_name): "AddExecutionEventsRequest", "AddExecutionEventsResponse", "AddTrialMeasurementRequest", + "AggregationOutput", + "AggregationResult", "Annotation", "AnnotationSpec", "ApiAuth", @@ -1270,6 +1290,7 @@ def _get_version(dependency_name): "AugmentPromptRequest", "AugmentPromptResponse", "AutomaticResources", + "AutoraterConfig", "AutoscalingMetricSpec", "AvroSource", "BatchCancelPipelineJobsOperationMetadata", @@ -1338,11 +1359,13 @@ def _get_version(dependency_name): "CometSpec", "CompleteTrialRequest", "CompletionStats", + "ComputationBasedMetricSpec", "ComputeTokensRequest", "ComputeTokensResponse", "ContainerRegistryDestination", "ContainerSpec", "Content", + "ContentMap", "Context", "CopyModelOperationMetadata", "CopyModelRequest", @@ -1419,6 +1442,8 @@ def _get_version(dependency_name): "CsvSource", "CustomJob", "CustomJobSpec", + "CustomOutput", + "CustomOutputFormatConfig", "DataFoundryServiceClient", "DataItem", "DataItemView", @@ -1513,10 +1538,14 @@ def _get_version(dependency_name): "EntityType", "EnvVar", "ErrorAnalysisAnnotation", + "EvaluateDatasetResponse", + "EvaluateDatasetRun", "EvaluateInstancesRequest", "EvaluateInstancesResponse", "EvaluatedAnnotation", "EvaluatedAnnotationExplanation", + "EvaluationConfig", + "EvaluationDataset", "EvaluationServiceClient", "Event", "EventActions", @@ -1704,6 +1733,7 @@ def _get_version(dependency_name): "JiraSource", "JobServiceClient", "JobState", + "LLMBasedMetricSpec", "LargeModelReference", "LineageSubgraph", "ListAnnotationsRequest", @@ -1828,6 +1858,8 @@ def _get_version(dependency_name): "MetadataSchema", "MetadataServiceClient", "MetadataStore", + "Metric", + "MetricResult", "MetricxInput", "MetricxInstance", "MetricxResult", @@ -1885,7 +1917,9 @@ def _get_version(dependency_name): "NotebookRuntimeType", "NotebookServiceClient", "NotebookSoftwareConfig", + "OutputConfig", "OutputFieldSpec", + "OutputInfo", "PSCAutomationConfig", "PSCAutomationState", "PairwiseChoice", @@ -1924,6 +1958,7 @@ def _get_version(dependency_name): "PostStartupScriptConfig", "PreTunedModel", "PrebuiltVoiceConfig", + "PredefinedMetricSpec", "PredefinedSplit", "PredictRequest", "PredictRequestResponseLoggingConfig", @@ -1984,6 +2019,7 @@ def _get_version(dependency_name): "RagQuery", "RagRetrievalConfig", "RagVectorDbConfig", + "RawOutput", "RawPredictRequest", "RayLogsSpec", "RayMetricSpec", diff --git a/google/cloud/aiplatform_v1/services/migration_service/client.py b/google/cloud/aiplatform_v1/services/migration_service/client.py index 074244583d..5d14719a74 100644 --- a/google/cloud/aiplatform_v1/services/migration_service/client.py +++ b/google/cloud/aiplatform_v1/services/migration_service/client.py @@ -293,40 +293,40 @@ def parse_dataset_path(path: str) -> Dict[str, str]: @staticmethod def dataset_path( project: str, + location: str, dataset: str, ) -> str: """Returns a fully-qualified dataset string.""" - return "projects/{project}/datasets/{dataset}".format( + return "projects/{project}/locations/{location}/datasets/{dataset}".format( project=project, + location=location, dataset=dataset, ) @staticmethod def parse_dataset_path(path: str) -> Dict[str, str]: """Parses a dataset path into its component segments.""" - m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)$", path) + m = re.match( + r"^projects/(?P.+?)/locations/(?P.+?)/datasets/(?P.+?)$", + path, + ) return m.groupdict() if m else {} @staticmethod def dataset_path( project: str, - location: str, dataset: str, ) -> str: """Returns a fully-qualified dataset string.""" - return "projects/{project}/locations/{location}/datasets/{dataset}".format( + return "projects/{project}/datasets/{dataset}".format( project=project, - location=location, dataset=dataset, ) @staticmethod def parse_dataset_path(path: str) -> Dict[str, str]: """Parses a dataset path into its component segments.""" - m = re.match( - r"^projects/(?P.+?)/locations/(?P.+?)/datasets/(?P.+?)$", - path, - ) + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)$", path) return m.groupdict() if m else {} @staticmethod diff --git a/google/cloud/aiplatform_v1/types/__init__.py b/google/cloud/aiplatform_v1/types/__init__.py index a71bd56311..0132ffa14a 100644 --- a/google/cloud/aiplatform_v1/types/__init__.py +++ b/google/cloud/aiplatform_v1/types/__init__.py @@ -214,6 +214,9 @@ EvaluatedAnnotationExplanation, ) from .evaluation_service import ( + AggregationOutput, + AggregationResult, + AutoraterConfig, BleuInput, BleuInstance, BleuMetricValue, @@ -227,8 +230,14 @@ CometInstance, CometResult, CometSpec, + ComputationBasedMetricSpec, + ContentMap, + CustomOutput, + CustomOutputFormatConfig, + EvaluateDatasetResponse, EvaluateInstancesRequest, EvaluateInstancesResponse, + EvaluationDataset, ExactMatchInput, ExactMatchInstance, ExactMatchMetricValue, @@ -246,10 +255,15 @@ GroundednessInstance, GroundednessResult, GroundednessSpec, + LLMBasedMetricSpec, + Metric, + MetricResult, MetricxInput, MetricxInstance, MetricxResult, MetricxSpec, + OutputConfig, + OutputInfo, PairwiseMetricInput, PairwiseMetricInstance, PairwiseMetricResult, @@ -266,6 +280,7 @@ PointwiseMetricInstance, PointwiseMetricResult, PointwiseMetricSpec, + PredefinedMetricSpec, QuestionAnsweringCorrectnessInput, QuestionAnsweringCorrectnessInstance, QuestionAnsweringCorrectnessResult, @@ -282,6 +297,7 @@ QuestionAnsweringRelevanceInstance, QuestionAnsweringRelevanceResult, QuestionAnsweringRelevanceSpec, + RawOutput, RougeInput, RougeInstance, RougeMetricValue, @@ -1136,6 +1152,8 @@ TrainingPipeline, ) from .tuning_job import ( + EvaluateDatasetRun, + EvaluationConfig, PreTunedModel, SupervisedHyperParameters, SupervisedTuningDatasetDistribution, @@ -1392,6 +1410,9 @@ "ErrorAnalysisAnnotation", "EvaluatedAnnotation", "EvaluatedAnnotationExplanation", + "AggregationOutput", + "AggregationResult", + "AutoraterConfig", "BleuInput", "BleuInstance", "BleuMetricValue", @@ -1405,8 +1426,14 @@ "CometInstance", "CometResult", "CometSpec", + "ComputationBasedMetricSpec", + "ContentMap", + "CustomOutput", + "CustomOutputFormatConfig", + "EvaluateDatasetResponse", "EvaluateInstancesRequest", "EvaluateInstancesResponse", + "EvaluationDataset", "ExactMatchInput", "ExactMatchInstance", "ExactMatchMetricValue", @@ -1424,10 +1451,15 @@ "GroundednessInstance", "GroundednessResult", "GroundednessSpec", + "LLMBasedMetricSpec", + "Metric", + "MetricResult", "MetricxInput", "MetricxInstance", "MetricxResult", "MetricxSpec", + "OutputConfig", + "OutputInfo", "PairwiseMetricInput", "PairwiseMetricInstance", "PairwiseMetricResult", @@ -1444,6 +1476,7 @@ "PointwiseMetricInstance", "PointwiseMetricResult", "PointwiseMetricSpec", + "PredefinedMetricSpec", "QuestionAnsweringCorrectnessInput", "QuestionAnsweringCorrectnessInstance", "QuestionAnsweringCorrectnessResult", @@ -1460,6 +1493,7 @@ "QuestionAnsweringRelevanceInstance", "QuestionAnsweringRelevanceResult", "QuestionAnsweringRelevanceSpec", + "RawOutput", "RougeInput", "RougeInstance", "RougeMetricValue", @@ -2142,6 +2176,8 @@ "StratifiedSplit", "TimestampSplit", "TrainingPipeline", + "EvaluateDatasetRun", + "EvaluationConfig", "PreTunedModel", "SupervisedHyperParameters", "SupervisedTuningDatasetDistribution", diff --git a/google/cloud/aiplatform_v1/types/evaluation_service.py b/google/cloud/aiplatform_v1/types/evaluation_service.py index 4518f716e4..9ba05390ac 100644 --- a/google/cloud/aiplatform_v1/types/evaluation_service.py +++ b/google/cloud/aiplatform_v1/types/evaluation_service.py @@ -19,13 +19,30 @@ import proto # type: ignore +from google.cloud.aiplatform_v1.types import content +from google.cloud.aiplatform_v1.types import io +import google.protobuf.struct_pb2 as struct_pb2 # type: ignore +import google.rpc.status_pb2 as status_pb2 # type: ignore + __protobuf__ = proto.module( package="google.cloud.aiplatform.v1", manifest={ "PairwiseChoice", "EvaluateInstancesRequest", + "Metric", + "AutoraterConfig", "EvaluateInstancesResponse", + "MetricResult", + "OutputConfig", + "EvaluationDataset", + "EvaluateDatasetResponse", + "OutputInfo", + "AggregationOutput", + "AggregationResult", + "PredefinedMetricSpec", + "ComputationBasedMetricSpec", + "LLMBasedMetricSpec", "ExactMatchInput", "ExactMatchInstance", "ExactMatchSpec", @@ -100,7 +117,10 @@ "PointwiseMetricInput", "PointwiseMetricInstance", "PointwiseMetricSpec", + "CustomOutputFormatConfig", "PointwiseMetricResult", + "CustomOutput", + "RawOutput", "PairwiseMetricInput", "PairwiseMetricInstance", "PairwiseMetricSpec", @@ -133,6 +153,7 @@ "MetricxSpec", "MetricxInstance", "MetricxResult", + "ContentMap", }, ) @@ -450,6 +471,218 @@ class EvaluateInstancesRequest(proto.Message): ) +class Metric(proto.Message): + r"""The metric used for running evaluations. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + predefined_metric_spec (google.cloud.aiplatform_v1.types.PredefinedMetricSpec): + The spec for a pre-defined metric. + + This field is a member of `oneof`_ ``metric_spec``. + computation_based_metric_spec (google.cloud.aiplatform_v1.types.ComputationBasedMetricSpec): + Spec for a computation based metric. + + This field is a member of `oneof`_ ``metric_spec``. + llm_based_metric_spec (google.cloud.aiplatform_v1.types.LLMBasedMetricSpec): + Spec for an LLM based metric. + + This field is a member of `oneof`_ ``metric_spec``. + pointwise_metric_spec (google.cloud.aiplatform_v1.types.PointwiseMetricSpec): + Spec for pointwise metric. + + This field is a member of `oneof`_ ``metric_spec``. + pairwise_metric_spec (google.cloud.aiplatform_v1.types.PairwiseMetricSpec): + Spec for pairwise metric. + + This field is a member of `oneof`_ ``metric_spec``. + exact_match_spec (google.cloud.aiplatform_v1.types.ExactMatchSpec): + Spec for exact match metric. + + This field is a member of `oneof`_ ``metric_spec``. + bleu_spec (google.cloud.aiplatform_v1.types.BleuSpec): + Spec for bleu metric. + + This field is a member of `oneof`_ ``metric_spec``. + rouge_spec (google.cloud.aiplatform_v1.types.RougeSpec): + Spec for rouge metric. + + This field is a member of `oneof`_ ``metric_spec``. + aggregation_metrics (MutableSequence[google.cloud.aiplatform_v1.types.Metric.AggregationMetric]): + Optional. The aggregation metrics to use. + """ + + class AggregationMetric(proto.Enum): + r"""The per-metric statistics on evaluation results supported by + ``EvaluationService.EvaluateDataset``. + + Values: + AGGREGATION_METRIC_UNSPECIFIED (0): + Unspecified aggregation metric. + AVERAGE (1): + Average aggregation metric. Not supported for + Pairwise metric. + MODE (2): + Mode aggregation metric. + STANDARD_DEVIATION (3): + Standard deviation aggregation metric. Not + supported for pairwise metric. + VARIANCE (4): + Variance aggregation metric. Not supported + for pairwise metric. + MINIMUM (5): + Minimum aggregation metric. Not supported for + pairwise metric. + MAXIMUM (6): + Maximum aggregation metric. Not supported for + pairwise metric. + MEDIAN (7): + Median aggregation metric. Not supported for + pairwise metric. + PERCENTILE_P90 (8): + 90th percentile aggregation metric. Not + supported for pairwise metric. + PERCENTILE_P95 (9): + 95th percentile aggregation metric. Not + supported for pairwise metric. + PERCENTILE_P99 (10): + 99th percentile aggregation metric. Not + supported for pairwise metric. + """ + + AGGREGATION_METRIC_UNSPECIFIED = 0 + AVERAGE = 1 + MODE = 2 + STANDARD_DEVIATION = 3 + VARIANCE = 4 + MINIMUM = 5 + MAXIMUM = 6 + MEDIAN = 7 + PERCENTILE_P90 = 8 + PERCENTILE_P95 = 9 + PERCENTILE_P99 = 10 + + predefined_metric_spec: "PredefinedMetricSpec" = proto.Field( + proto.MESSAGE, + number=8, + oneof="metric_spec", + message="PredefinedMetricSpec", + ) + computation_based_metric_spec: "ComputationBasedMetricSpec" = proto.Field( + proto.MESSAGE, + number=9, + oneof="metric_spec", + message="ComputationBasedMetricSpec", + ) + llm_based_metric_spec: "LLMBasedMetricSpec" = proto.Field( + proto.MESSAGE, + number=10, + oneof="metric_spec", + message="LLMBasedMetricSpec", + ) + pointwise_metric_spec: "PointwiseMetricSpec" = proto.Field( + proto.MESSAGE, + number=2, + oneof="metric_spec", + message="PointwiseMetricSpec", + ) + pairwise_metric_spec: "PairwiseMetricSpec" = proto.Field( + proto.MESSAGE, + number=3, + oneof="metric_spec", + message="PairwiseMetricSpec", + ) + exact_match_spec: "ExactMatchSpec" = proto.Field( + proto.MESSAGE, + number=4, + oneof="metric_spec", + message="ExactMatchSpec", + ) + bleu_spec: "BleuSpec" = proto.Field( + proto.MESSAGE, + number=5, + oneof="metric_spec", + message="BleuSpec", + ) + rouge_spec: "RougeSpec" = proto.Field( + proto.MESSAGE, + number=6, + oneof="metric_spec", + message="RougeSpec", + ) + aggregation_metrics: MutableSequence[AggregationMetric] = proto.RepeatedField( + proto.ENUM, + number=1, + enum=AggregationMetric, + ) + + +class AutoraterConfig(proto.Message): + r"""The configs for autorater. This is applicable to both + EvaluateInstances and EvaluateDataset. + + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + sampling_count (int): + Optional. Number of samples for each instance + in the dataset. If not specified, the default is + 4. Minimum value is 1, maximum value is 32. + + This field is a member of `oneof`_ ``_sampling_count``. + flip_enabled (bool): + Optional. Default is true. Whether to flip the candidate and + baseline responses. This is only applicable to the pairwise + metric. If enabled, also provide + PairwiseMetricSpec.candidate_response_field_name and + PairwiseMetricSpec.baseline_response_field_name. When + rendering PairwiseMetricSpec.metric_prompt_template, the + candidate and baseline fields will be flipped for half of + the samples to reduce bias. + + This field is a member of `oneof`_ ``_flip_enabled``. + autorater_model (str): + Optional. The fully qualified name of the publisher model or + tuned autorater endpoint to use. + + Publisher model format: + ``projects/{project}/locations/{location}/publishers/*/models/*`` + + Tuned model endpoint format: + ``projects/{project}/locations/{location}/endpoints/{endpoint}`` + generation_config (google.cloud.aiplatform_v1.types.GenerationConfig): + Optional. Configuration options for model + generation and outputs. + """ + + sampling_count: int = proto.Field( + proto.INT32, + number=1, + optional=True, + ) + flip_enabled: bool = proto.Field( + proto.BOOL, + number=2, + optional=True, + ) + autorater_model: str = proto.Field( + proto.STRING, + number=3, + ) + generation_config: content.GenerationConfig = proto.Field( + proto.MESSAGE, + number=4, + message=content.GenerationConfig, + ) + + class EvaluateInstancesResponse(proto.Message): r"""Response message for EvaluationService.EvaluateInstances. @@ -575,6 +808,11 @@ class EvaluateInstancesResponse(proto.Message): Result for Metricx metric. This field is a member of `oneof`_ ``evaluation_results``. + metric_results (MutableSequence[google.cloud.aiplatform_v1.types.MetricResult]): + Metric results for each instance. + The order of the metric results is guaranteed to + be the same as the order of the instances in the + request. """ exact_match_results: "ExactMatchResults" = proto.Field( @@ -737,6 +975,407 @@ class EvaluateInstancesResponse(proto.Message): oneof="evaluation_results", message="MetricxResult", ) + metric_results: MutableSequence["MetricResult"] = proto.RepeatedField( + proto.MESSAGE, + number=43, + message="MetricResult", + ) + + +class MetricResult(proto.Message): + r"""Result for a single metric on a single instance. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + score (float): + Output only. The score for the metric. + Please refer to each metric's documentation for + the meaning of the score. + + This field is a member of `oneof`_ ``_score``. + explanation (str): + Output only. The explanation for the metric + result. + + This field is a member of `oneof`_ ``_explanation``. + error (google.rpc.status_pb2.Status): + Output only. The error status for the metric + result. + + This field is a member of `oneof`_ ``_error``. + """ + + score: float = proto.Field( + proto.FLOAT, + number=1, + optional=True, + ) + explanation: str = proto.Field( + proto.STRING, + number=3, + optional=True, + ) + error: status_pb2.Status = proto.Field( + proto.MESSAGE, + number=4, + optional=True, + message=status_pb2.Status, + ) + + +class OutputConfig(proto.Message): + r"""Config for evaluation output. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + gcs_destination (google.cloud.aiplatform_v1.types.GcsDestination): + Cloud storage destination for evaluation + output. + + This field is a member of `oneof`_ ``destination``. + """ + + gcs_destination: io.GcsDestination = proto.Field( + proto.MESSAGE, + number=1, + oneof="destination", + message=io.GcsDestination, + ) + + +class EvaluationDataset(proto.Message): + r"""The dataset used for evaluation. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + gcs_source (google.cloud.aiplatform_v1.types.GcsSource): + Cloud storage source holds the dataset. + Currently only one Cloud Storage file path is + supported. + + This field is a member of `oneof`_ ``source``. + bigquery_source (google.cloud.aiplatform_v1.types.BigQuerySource): + BigQuery source holds the dataset. + + This field is a member of `oneof`_ ``source``. + """ + + gcs_source: io.GcsSource = proto.Field( + proto.MESSAGE, + number=1, + oneof="source", + message=io.GcsSource, + ) + bigquery_source: io.BigQuerySource = proto.Field( + proto.MESSAGE, + number=2, + oneof="source", + message=io.BigQuerySource, + ) + + +class EvaluateDatasetResponse(proto.Message): + r"""The results from an evaluation run performed by the + EvaluationService. + + Attributes: + aggregation_output (google.cloud.aiplatform_v1.types.AggregationOutput): + Output only. Aggregation statistics derived + from results of EvaluationService. + output_info (google.cloud.aiplatform_v1.types.OutputInfo): + Output only. Output info for + EvaluationService. + """ + + aggregation_output: "AggregationOutput" = proto.Field( + proto.MESSAGE, + number=1, + message="AggregationOutput", + ) + output_info: "OutputInfo" = proto.Field( + proto.MESSAGE, + number=3, + message="OutputInfo", + ) + + +class OutputInfo(proto.Message): + r"""Describes the info for output of EvaluationService. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + gcs_output_directory (str): + Output only. The full path of the Cloud + Storage directory created, into which the + evaluation results and aggregation results are + written. + + This field is a member of `oneof`_ ``output_location``. + """ + + gcs_output_directory: str = proto.Field( + proto.STRING, + number=1, + oneof="output_location", + ) + + +class AggregationOutput(proto.Message): + r"""The aggregation result for the entire dataset and all + metrics. + + Attributes: + dataset (google.cloud.aiplatform_v1.types.EvaluationDataset): + The dataset used for evaluation & + aggregation. + aggregation_results (MutableSequence[google.cloud.aiplatform_v1.types.AggregationResult]): + One AggregationResult per metric. + """ + + dataset: "EvaluationDataset" = proto.Field( + proto.MESSAGE, + number=1, + message="EvaluationDataset", + ) + aggregation_results: MutableSequence["AggregationResult"] = proto.RepeatedField( + proto.MESSAGE, + number=2, + message="AggregationResult", + ) + + +class AggregationResult(proto.Message): + r"""The aggregation result for a single metric. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + pointwise_metric_result (google.cloud.aiplatform_v1.types.PointwiseMetricResult): + Result for pointwise metric. + + This field is a member of `oneof`_ ``aggregation_result``. + pairwise_metric_result (google.cloud.aiplatform_v1.types.PairwiseMetricResult): + Result for pairwise metric. + + This field is a member of `oneof`_ ``aggregation_result``. + exact_match_metric_value (google.cloud.aiplatform_v1.types.ExactMatchMetricValue): + Results for exact match metric. + + This field is a member of `oneof`_ ``aggregation_result``. + bleu_metric_value (google.cloud.aiplatform_v1.types.BleuMetricValue): + Results for bleu metric. + + This field is a member of `oneof`_ ``aggregation_result``. + rouge_metric_value (google.cloud.aiplatform_v1.types.RougeMetricValue): + Results for rouge metric. + + This field is a member of `oneof`_ ``aggregation_result``. + aggregation_metric (google.cloud.aiplatform_v1.types.Metric.AggregationMetric): + Aggregation metric. + """ + + pointwise_metric_result: "PointwiseMetricResult" = proto.Field( + proto.MESSAGE, + number=5, + oneof="aggregation_result", + message="PointwiseMetricResult", + ) + pairwise_metric_result: "PairwiseMetricResult" = proto.Field( + proto.MESSAGE, + number=6, + oneof="aggregation_result", + message="PairwiseMetricResult", + ) + exact_match_metric_value: "ExactMatchMetricValue" = proto.Field( + proto.MESSAGE, + number=7, + oneof="aggregation_result", + message="ExactMatchMetricValue", + ) + bleu_metric_value: "BleuMetricValue" = proto.Field( + proto.MESSAGE, + number=8, + oneof="aggregation_result", + message="BleuMetricValue", + ) + rouge_metric_value: "RougeMetricValue" = proto.Field( + proto.MESSAGE, + number=9, + oneof="aggregation_result", + message="RougeMetricValue", + ) + aggregation_metric: "Metric.AggregationMetric" = proto.Field( + proto.ENUM, + number=4, + enum="Metric.AggregationMetric", + ) + + +class PredefinedMetricSpec(proto.Message): + r"""The spec for a pre-defined metric. + + Attributes: + metric_spec_name (str): + Required. The name of a pre-defined metric, such as + "instruction_following_v1" or "text_quality_v1". + metric_spec_parameters (google.protobuf.struct_pb2.Struct): + Optional. The parameters needed to run the + pre-defined metric. + """ + + metric_spec_name: str = proto.Field( + proto.STRING, + number=1, + ) + metric_spec_parameters: struct_pb2.Struct = proto.Field( + proto.MESSAGE, + number=2, + message=struct_pb2.Struct, + ) + + +class ComputationBasedMetricSpec(proto.Message): + r"""Specification for a computation based metric. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + type_ (google.cloud.aiplatform_v1.types.ComputationBasedMetricSpec.ComputationBasedMetricType): + Required. The type of the computation based + metric. + + This field is a member of `oneof`_ ``_type``. + parameters (google.protobuf.struct_pb2.Struct): + Optional. A map of parameters for the metric, e.g. + {"rouge_type": "rougeL"}. + + This field is a member of `oneof`_ ``_parameters``. + """ + + class ComputationBasedMetricType(proto.Enum): + r"""Types of computation based metrics. + + Values: + COMPUTATION_BASED_METRIC_TYPE_UNSPECIFIED (0): + Unspecified computation based metric type. + EXACT_MATCH (1): + Exact match metric. + BLEU (2): + BLEU metric. + ROUGE (3): + ROUGE metric. + """ + + COMPUTATION_BASED_METRIC_TYPE_UNSPECIFIED = 0 + EXACT_MATCH = 1 + BLEU = 2 + ROUGE = 3 + + type_: ComputationBasedMetricType = proto.Field( + proto.ENUM, + number=1, + optional=True, + enum=ComputationBasedMetricType, + ) + parameters: struct_pb2.Struct = proto.Field( + proto.MESSAGE, + number=2, + optional=True, + message=struct_pb2.Struct, + ) + + +class LLMBasedMetricSpec(proto.Message): + r"""Specification for an LLM based metric. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + rubric_group_key (str): + Use a pre-defined group of rubrics associated with the + input. Refers to a key in the rubric_groups map of + EvaluationInstance. + + This field is a member of `oneof`_ ``rubrics_source``. + predefined_rubric_generation_spec (google.cloud.aiplatform_v1.types.PredefinedMetricSpec): + Dynamically generate rubrics using a + predefined spec. + + This field is a member of `oneof`_ ``rubrics_source``. + metric_prompt_template (str): + Required. Template for the prompt sent to the + judge model. + + This field is a member of `oneof`_ ``_metric_prompt_template``. + system_instruction (str): + Optional. System instructions for the judge + model. + + This field is a member of `oneof`_ ``_system_instruction``. + judge_autorater_config (google.cloud.aiplatform_v1.types.AutoraterConfig): + Optional. Optional configuration for the + judge LLM (Autorater). + + This field is a member of `oneof`_ ``_judge_autorater_config``. + additional_config (google.protobuf.struct_pb2.Struct): + Optional. Optional additional configuration + for the metric. + + This field is a member of `oneof`_ ``_additional_config``. + """ + + rubric_group_key: str = proto.Field( + proto.STRING, + number=4, + oneof="rubrics_source", + ) + predefined_rubric_generation_spec: "PredefinedMetricSpec" = proto.Field( + proto.MESSAGE, + number=6, + oneof="rubrics_source", + message="PredefinedMetricSpec", + ) + metric_prompt_template: str = proto.Field( + proto.STRING, + number=1, + optional=True, + ) + system_instruction: str = proto.Field( + proto.STRING, + number=2, + optional=True, + ) + judge_autorater_config: "AutoraterConfig" = proto.Field( + proto.MESSAGE, + number=3, + optional=True, + message="AutoraterConfig", + ) + additional_config: struct_pb2.Struct = proto.Field( + proto.MESSAGE, + number=7, + optional=True, + message=struct_pb2.Struct, + ) class ExactMatchInput(proto.Message): @@ -2735,6 +3374,10 @@ class PointwiseMetricInstance(proto.Message): r"""Pointwise metric instance. Usually one instance corresponds to one row in an evaluation dataset. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields @@ -2744,6 +3387,14 @@ class PointwiseMetricInstance(proto.Message): are expected in the json_instance to render PointwiseMetricSpec.instance_prompt_template. + This field is a member of `oneof`_ ``instance``. + content_map_instance (google.cloud.aiplatform_v1.types.ContentMap): + Key-value contents for the mutlimodality + input, including text, image, video, audio, and + pdf, etc. The key is placeholder in metric + prompt template, and the value is the multimodal + content. + This field is a member of `oneof`_ ``instance``. """ @@ -2752,6 +3403,12 @@ class PointwiseMetricInstance(proto.Message): number=1, oneof="instance", ) + content_map_instance: "ContentMap" = proto.Field( + proto.MESSAGE, + number=2, + oneof="instance", + message="ContentMap", + ) class PointwiseMetricSpec(proto.Message): @@ -2765,6 +3422,21 @@ class PointwiseMetricSpec(proto.Message): pointwise metric. This field is a member of `oneof`_ ``_metric_prompt_template``. + system_instruction (str): + Optional. System instructions for pointwise + metric. + + This field is a member of `oneof`_ ``_system_instruction``. + custom_output_format_config (google.cloud.aiplatform_v1.types.CustomOutputFormatConfig): + Optional. CustomOutputFormatConfig allows customization of + metric output. By default, metrics return a score and + explanation. When this config is set, the default output is + replaced with either: + + - The raw output string. + - A parsed output based on a user-defined schema. If a + custom format is chosen, the ``score`` and ``explanation`` + fields in the corresponding metric result will be empty. """ metric_prompt_template: str = proto.Field( @@ -2772,6 +3444,35 @@ class PointwiseMetricSpec(proto.Message): number=1, optional=True, ) + system_instruction: str = proto.Field( + proto.STRING, + number=2, + optional=True, + ) + custom_output_format_config: "CustomOutputFormatConfig" = proto.Field( + proto.MESSAGE, + number=3, + message="CustomOutputFormatConfig", + ) + + +class CustomOutputFormatConfig(proto.Message): + r"""Spec for custom output format configuration. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + return_raw_output (bool): + Optional. Whether to return raw output. + + This field is a member of `oneof`_ ``custom_output_format_config``. + """ + + return_raw_output: bool = proto.Field( + proto.BOOL, + number=1, + oneof="custom_output_format_config", + ) class PointwiseMetricResult(proto.Message): @@ -2787,6 +3488,8 @@ class PointwiseMetricResult(proto.Message): explanation (str): Output only. Explanation for pointwise metric score. + custom_output (google.cloud.aiplatform_v1.types.CustomOutput): + Output only. Spec for custom output. """ score: float = proto.Field( @@ -2798,6 +3501,45 @@ class PointwiseMetricResult(proto.Message): proto.STRING, number=2, ) + custom_output: "CustomOutput" = proto.Field( + proto.MESSAGE, + number=3, + message="CustomOutput", + ) + + +class CustomOutput(proto.Message): + r"""Spec for custom output. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + raw_outputs (google.cloud.aiplatform_v1.types.RawOutput): + Output only. List of raw output strings. + + This field is a member of `oneof`_ ``custom_output``. + """ + + raw_outputs: "RawOutput" = proto.Field( + proto.MESSAGE, + number=1, + oneof="custom_output", + message="RawOutput", + ) + + +class RawOutput(proto.Message): + r"""Raw output. + + Attributes: + raw_output (MutableSequence[str]): + Output only. Raw output string. + """ + + raw_output: MutableSequence[str] = proto.RepeatedField( + proto.STRING, + number=1, + ) class PairwiseMetricInput(proto.Message): @@ -2826,6 +3568,10 @@ class PairwiseMetricInstance(proto.Message): r"""Pairwise metric instance. Usually one instance corresponds to one row in an evaluation dataset. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields @@ -2835,6 +3581,14 @@ class PairwiseMetricInstance(proto.Message): are expected in the json_instance to render PairwiseMetricSpec.instance_prompt_template. + This field is a member of `oneof`_ ``instance``. + content_map_instance (google.cloud.aiplatform_v1.types.ContentMap): + Key-value contents for the mutlimodality + input, including text, image, video, audio, and + pdf, etc. The key is placeholder in metric + prompt template, and the value is the multimodal + content. + This field is a member of `oneof`_ ``instance``. """ @@ -2843,6 +3597,12 @@ class PairwiseMetricInstance(proto.Message): number=1, oneof="instance", ) + content_map_instance: "ContentMap" = proto.Field( + proto.MESSAGE, + number=2, + oneof="instance", + message="ContentMap", + ) class PairwiseMetricSpec(proto.Message): @@ -2856,6 +3616,23 @@ class PairwiseMetricSpec(proto.Message): metric. This field is a member of `oneof`_ ``_metric_prompt_template``. + candidate_response_field_name (str): + Optional. The field name of the candidate + response. + baseline_response_field_name (str): + Optional. The field name of the baseline + response. + system_instruction (str): + Optional. System instructions for pairwise + metric. + + This field is a member of `oneof`_ ``_system_instruction``. + custom_output_format_config (google.cloud.aiplatform_v1.types.CustomOutputFormatConfig): + Optional. CustomOutputFormatConfig allows customization of + metric output. When this config is set, the default output + is replaced with the raw output string. If a custom format + is chosen, the ``pairwise_choice`` and ``explanation`` + fields in the corresponding metric result will be empty. """ metric_prompt_template: str = proto.Field( @@ -2863,6 +3640,24 @@ class PairwiseMetricSpec(proto.Message): number=1, optional=True, ) + candidate_response_field_name: str = proto.Field( + proto.STRING, + number=2, + ) + baseline_response_field_name: str = proto.Field( + proto.STRING, + number=3, + ) + system_instruction: str = proto.Field( + proto.STRING, + number=4, + optional=True, + ) + custom_output_format_config: "CustomOutputFormatConfig" = proto.Field( + proto.MESSAGE, + number=5, + message="CustomOutputFormatConfig", + ) class PairwiseMetricResult(proto.Message): @@ -2874,6 +3669,8 @@ class PairwiseMetricResult(proto.Message): explanation (str): Output only. Explanation for pairwise metric score. + custom_output (google.cloud.aiplatform_v1.types.CustomOutput): + Output only. Spec for custom output. """ pairwise_choice: "PairwiseChoice" = proto.Field( @@ -2885,6 +3682,11 @@ class PairwiseMetricResult(proto.Message): proto.STRING, number=2, ) + custom_output: "CustomOutput" = proto.Field( + proto.MESSAGE, + number=3, + message="CustomOutput", + ) class ToolCallValidInput(proto.Message): @@ -3541,4 +4343,35 @@ class MetricxResult(proto.Message): ) +class ContentMap(proto.Message): + r"""Map of placeholder in metric prompt template to contents of + model input. + + Attributes: + values (MutableMapping[str, google.cloud.aiplatform_v1.types.ContentMap.Contents]): + Optional. Map of placeholder to contents. + """ + + class Contents(proto.Message): + r"""Repeated Content type. + + Attributes: + contents (MutableSequence[google.cloud.aiplatform_v1.types.Content]): + Optional. Repeated contents. + """ + + contents: MutableSequence[content.Content] = proto.RepeatedField( + proto.MESSAGE, + number=1, + message=content.Content, + ) + + values: MutableMapping[str, Contents] = proto.MapField( + proto.STRING, + proto.MESSAGE, + number=1, + message=Contents, + ) + + __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/aiplatform_v1/types/tuning_job.py b/google/cloud/aiplatform_v1/types/tuning_job.py index 49d71ba0de..278063f2f1 100644 --- a/google/cloud/aiplatform_v1/types/tuning_job.py +++ b/google/cloud/aiplatform_v1/types/tuning_job.py @@ -21,6 +21,7 @@ from google.cloud.aiplatform_v1.types import content from google.cloud.aiplatform_v1.types import encryption_spec as gca_encryption_spec +from google.cloud.aiplatform_v1.types import evaluation_service from google.cloud.aiplatform_v1.types import job_state import google.protobuf.timestamp_pb2 as timestamp_pb2 # type: ignore import google.rpc.status_pb2 as status_pb2 # type: ignore @@ -37,6 +38,8 @@ "SupervisedHyperParameters", "SupervisedTuningSpec", "TunedModelRef", + "EvaluationConfig", + "EvaluateDatasetRun", "TunedModelCheckpoint", "PreTunedModel", }, @@ -141,6 +144,9 @@ class TuningJob(proto.Message): Users starting the pipeline must have the ``iam.serviceAccounts.actAs`` permission on this service account. + evaluate_dataset_runs (MutableSequence[google.cloud.aiplatform_v1.types.EvaluateDatasetRun]): + Output only. Evaluation runs for the Tuning + Job. """ base_model: str = proto.Field( @@ -230,6 +236,11 @@ class TuningJob(proto.Message): proto.STRING, number=22, ) + evaluate_dataset_runs: MutableSequence["EvaluateDatasetRun"] = proto.RepeatedField( + proto.MESSAGE, + number=32, + message="EvaluateDatasetRun", + ) class TunedModel(proto.Message): @@ -576,6 +587,8 @@ class SupervisedTuningSpec(proto.Message): last checkpoint will be exported. Otherwise, enable intermediate checkpoints for SFT. Default is false. + evaluation_config (google.cloud.aiplatform_v1.types.EvaluationConfig): + Optional. Evaluation Config for Tuning Job. """ training_dataset_uri: str = proto.Field( @@ -595,6 +608,11 @@ class SupervisedTuningSpec(proto.Message): proto.BOOL, number=6, ) + evaluation_config: "EvaluationConfig" = proto.Field( + proto.MESSAGE, + number=5, + message="EvaluationConfig", + ) class TunedModelRef(proto.Message): @@ -641,6 +659,90 @@ class TunedModelRef(proto.Message): ) +class EvaluationConfig(proto.Message): + r"""Evaluation Config for Tuning Job. + + Attributes: + metrics (MutableSequence[google.cloud.aiplatform_v1.types.Metric]): + Required. The metrics used for evaluation. + output_config (google.cloud.aiplatform_v1.types.OutputConfig): + Required. Config for evaluation output. + autorater_config (google.cloud.aiplatform_v1.types.AutoraterConfig): + Optional. Autorater config for evaluation. + inference_generation_config (google.cloud.aiplatform_v1.types.GenerationConfig): + Optional. Configuration options for inference + generation and outputs. If not set, default + generation parameters are used. + """ + + metrics: MutableSequence[evaluation_service.Metric] = proto.RepeatedField( + proto.MESSAGE, + number=1, + message=evaluation_service.Metric, + ) + output_config: evaluation_service.OutputConfig = proto.Field( + proto.MESSAGE, + number=2, + message=evaluation_service.OutputConfig, + ) + autorater_config: evaluation_service.AutoraterConfig = proto.Field( + proto.MESSAGE, + number=3, + message=evaluation_service.AutoraterConfig, + ) + inference_generation_config: content.GenerationConfig = proto.Field( + proto.MESSAGE, + number=5, + message=content.GenerationConfig, + ) + + +class EvaluateDatasetRun(proto.Message): + r"""Evaluate Dataset Run Result for Tuning Job. + + Attributes: + operation_name (str): + Output only. Deprecated: The updated architecture uses + evaluation_run instead. + evaluation_run (str): + Output only. The resource name of the evaluation run. + Format: + ``projects/{project}/locations/{location}/evaluationRuns/{evaluation_run_id}``. + checkpoint_id (str): + Output only. The checkpoint id used in the + evaluation run. Only populated when evaluating + checkpoints. + evaluate_dataset_response (google.cloud.aiplatform_v1.types.EvaluateDatasetResponse): + Output only. Results for EvaluationService. + error (google.rpc.status_pb2.Status): + Output only. The error of the evaluation run + if any. + """ + + operation_name: str = proto.Field( + proto.STRING, + number=1, + ) + evaluation_run: str = proto.Field( + proto.STRING, + number=5, + ) + checkpoint_id: str = proto.Field( + proto.STRING, + number=2, + ) + evaluate_dataset_response: evaluation_service.EvaluateDatasetResponse = proto.Field( + proto.MESSAGE, + number=3, + message=evaluation_service.EvaluateDatasetResponse, + ) + error: status_pb2.Status = proto.Field( + proto.MESSAGE, + number=4, + message=status_pb2.Status, + ) + + class TunedModelCheckpoint(proto.Message): r"""TunedModelCheckpoint for the Tuned Model of a Tuning Job. diff --git a/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json b/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json index 0b10c3efe3..00899eb0aa 100644 --- a/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json +++ b/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-aiplatform", - "version": "1.144.0" + "version": "0.0.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1beta1.json b/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1beta1.json index e3ba7ef751..ae87e2f5d6 100644 --- a/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1beta1.json +++ b/samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1beta1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-aiplatform", - "version": "1.144.0" + "version": "0.0.0" }, "snippets": [ { diff --git a/tests/unit/gapic/aiplatform_v1/test_evaluation_service.py b/tests/unit/gapic/aiplatform_v1/test_evaluation_service.py index ea77f183b2..3de3ad40fe 100644 --- a/tests/unit/gapic/aiplatform_v1/test_evaluation_service.py +++ b/tests/unit/gapic/aiplatform_v1/test_evaluation_service.py @@ -70,7 +70,9 @@ EvaluationServiceClient, ) from google.cloud.aiplatform_v1.services.evaluation_service import transports +from google.cloud.aiplatform_v1.types import content from google.cloud.aiplatform_v1.types import evaluation_service +from google.cloud.aiplatform_v1.types import tool from google.cloud.location import locations_pb2 from google.iam.v1 import iam_policy_pb2 # type: ignore from google.iam.v1 import options_pb2 # type: ignore @@ -78,6 +80,8 @@ from google.longrunning import operations_pb2 # type: ignore from google.oauth2 import service_account import google.auth +import google.protobuf.duration_pb2 as duration_pb2 # type: ignore +import google.protobuf.struct_pb2 as struct_pb2 # type: ignore CRED_INFO_JSON = { diff --git a/tests/unit/gapic/aiplatform_v1/test_gen_ai_tuning_service.py b/tests/unit/gapic/aiplatform_v1/test_gen_ai_tuning_service.py index e23e353b98..3438eb27d7 100644 --- a/tests/unit/gapic/aiplatform_v1/test_gen_ai_tuning_service.py +++ b/tests/unit/gapic/aiplatform_v1/test_gen_ai_tuning_service.py @@ -76,9 +76,11 @@ from google.cloud.aiplatform_v1.services.gen_ai_tuning_service import transports from google.cloud.aiplatform_v1.types import content from google.cloud.aiplatform_v1.types import encryption_spec +from google.cloud.aiplatform_v1.types import evaluation_service from google.cloud.aiplatform_v1.types import genai_tuning_service from google.cloud.aiplatform_v1.types import io from google.cloud.aiplatform_v1.types import job_state +from google.cloud.aiplatform_v1.types import openapi from google.cloud.aiplatform_v1.types import tool from google.cloud.aiplatform_v1.types import tuning_job from google.cloud.aiplatform_v1.types import tuning_job as gca_tuning_job @@ -4742,6 +4744,155 @@ def test_create_tuning_job_rest_call_success(request_type): "adapter_size": 1, }, "export_last_checkpoint_only": True, + "evaluation_config": { + "metrics": [ + { + "predefined_metric_spec": { + "metric_spec_name": "metric_spec_name_value", + "metric_spec_parameters": {"fields": {}}, + }, + "computation_based_metric_spec": {"type_": 1, "parameters": {}}, + "llm_based_metric_spec": { + "rubric_group_key": "rubric_group_key_value", + "predefined_rubric_generation_spec": {}, + "metric_prompt_template": "metric_prompt_template_value", + "system_instruction": "system_instruction_value", + "judge_autorater_config": { + "sampling_count": 1507, + "flip_enabled": True, + "autorater_model": "autorater_model_value", + "generation_config": { + "temperature": 0.1198, + "top_p": 0.546, + "top_k": 0.541, + "candidate_count": 1573, + "max_output_tokens": 1865, + "stop_sequences": [ + "stop_sequences_value1", + "stop_sequences_value2", + ], + "response_logprobs": True, + "logprobs": 872, + "presence_penalty": 0.1713, + "frequency_penalty": 0.18380000000000002, + "seed": 417, + "response_mime_type": "response_mime_type_value", + "response_schema": { + "type_": 1, + "format_": "format__value", + "title": "title_value", + "description": "description_value", + "nullable": True, + "default": { + "null_value": 0, + "number_value": 0.1285, + "string_value": "string_value_value", + "bool_value": True, + "struct_value": {}, + "list_value": {"values": {}}, + }, + "items": {}, + "min_items": 965, + "max_items": 967, + "enum": ["enum_value1", "enum_value2"], + "properties": {}, + "property_ordering": [ + "property_ordering_value1", + "property_ordering_value2", + ], + "required": [ + "required_value1", + "required_value2", + ], + "min_properties": 1520, + "max_properties": 1522, + "minimum": 0.764, + "maximum": 0.766, + "min_length": 1061, + "max_length": 1063, + "pattern": "pattern_value", + "example": {}, + "any_of": {}, + "additional_properties": {}, + "ref": "ref_value", + "defs": {}, + }, + "response_json_schema": {}, + "routing_config": { + "auto_mode": {"model_routing_preference": 1}, + "manual_mode": { + "model_name": "model_name_value" + }, + }, + "audio_timestamp": True, + "response_modalities": [1], + "media_resolution": 1, + "speech_config": { + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "voice_name_value" + }, + "replicated_voice_config": { + "mime_type": "mime_type_value", + "voice_sample_audio": b"voice_sample_audio_blob", + }, + }, + "language_code": "language_code_value", + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "speaker_value", + "voice_config": {}, + } + ] + }, + }, + "thinking_config": { + "include_thoughts": True, + "thinking_budget": 1590, + "thinking_level": 1, + }, + "image_config": { + "image_output_options": { + "mime_type": "mime_type_value", + "compression_quality": 2074, + }, + "aspect_ratio": "aspect_ratio_value", + "person_generation": 1, + "image_size": "image_size_value", + }, + }, + }, + "additional_config": {}, + }, + "pointwise_metric_spec": { + "metric_prompt_template": "metric_prompt_template_value", + "system_instruction": "system_instruction_value", + "custom_output_format_config": {"return_raw_output": True}, + }, + "pairwise_metric_spec": { + "metric_prompt_template": "metric_prompt_template_value", + "candidate_response_field_name": "candidate_response_field_name_value", + "baseline_response_field_name": "baseline_response_field_name_value", + "system_instruction": "system_instruction_value", + "custom_output_format_config": {}, + }, + "exact_match_spec": {}, + "bleu_spec": {"use_effective_order": True}, + "rouge_spec": { + "rouge_type": "rouge_type_value", + "use_stemmer": True, + "split_summaries": True, + }, + "aggregation_metrics": [1], + } + ], + "output_config": { + "gcs_destination": {"output_uri_prefix": "output_uri_prefix_value"} + }, + "autorater_config": {}, + "inference_generation_config": {}, + }, }, "name": "name_value", "tuned_model_display_name": "tuned_model_display_name_value", @@ -4811,7 +4962,7 @@ def test_create_tuning_job_rest_call_success(request_type): }, "function_call": { "name": "name_value", - "args": {"fields": {}}, + "args": {}, "partial_args": [ { "null_value": 0, @@ -4872,6 +5023,50 @@ def test_create_tuning_job_rest_call_success(request_type): }, "encryption_spec": {"kms_key_name": "kms_key_name_value"}, "service_account": "service_account_value", + "evaluate_dataset_runs": [ + { + "operation_name": "operation_name_value", + "evaluation_run": "evaluation_run_value", + "checkpoint_id": "checkpoint_id_value", + "evaluate_dataset_response": { + "aggregation_output": { + "dataset": { + "gcs_source": {"uris": ["uris_value1", "uris_value2"]}, + "bigquery_source": {"input_uri": "input_uri_value"}, + }, + "aggregation_results": [ + { + "pointwise_metric_result": { + "score": 0.54, + "explanation": "explanation_value", + "custom_output": { + "raw_outputs": { + "raw_output": [ + "raw_output_value1", + "raw_output_value2", + ] + } + }, + }, + "pairwise_metric_result": { + "pairwise_choice": 1, + "explanation": "explanation_value", + "custom_output": {}, + }, + "exact_match_metric_value": {"score": 0.54}, + "bleu_metric_value": {"score": 0.54}, + "rouge_metric_value": {"score": 0.54}, + "aggregation_metric": 1, + } + ], + }, + "output_info": { + "gcs_output_directory": "gcs_output_directory_value" + }, + }, + "error": {}, + } + ], } # The version of a generated dependency at test runtime may differ from the version used during generation. # Delete any fields which are not present in the current runtime dependency @@ -6375,6 +6570,155 @@ async def test_create_tuning_job_rest_asyncio_call_success(request_type): "adapter_size": 1, }, "export_last_checkpoint_only": True, + "evaluation_config": { + "metrics": [ + { + "predefined_metric_spec": { + "metric_spec_name": "metric_spec_name_value", + "metric_spec_parameters": {"fields": {}}, + }, + "computation_based_metric_spec": {"type_": 1, "parameters": {}}, + "llm_based_metric_spec": { + "rubric_group_key": "rubric_group_key_value", + "predefined_rubric_generation_spec": {}, + "metric_prompt_template": "metric_prompt_template_value", + "system_instruction": "system_instruction_value", + "judge_autorater_config": { + "sampling_count": 1507, + "flip_enabled": True, + "autorater_model": "autorater_model_value", + "generation_config": { + "temperature": 0.1198, + "top_p": 0.546, + "top_k": 0.541, + "candidate_count": 1573, + "max_output_tokens": 1865, + "stop_sequences": [ + "stop_sequences_value1", + "stop_sequences_value2", + ], + "response_logprobs": True, + "logprobs": 872, + "presence_penalty": 0.1713, + "frequency_penalty": 0.18380000000000002, + "seed": 417, + "response_mime_type": "response_mime_type_value", + "response_schema": { + "type_": 1, + "format_": "format__value", + "title": "title_value", + "description": "description_value", + "nullable": True, + "default": { + "null_value": 0, + "number_value": 0.1285, + "string_value": "string_value_value", + "bool_value": True, + "struct_value": {}, + "list_value": {"values": {}}, + }, + "items": {}, + "min_items": 965, + "max_items": 967, + "enum": ["enum_value1", "enum_value2"], + "properties": {}, + "property_ordering": [ + "property_ordering_value1", + "property_ordering_value2", + ], + "required": [ + "required_value1", + "required_value2", + ], + "min_properties": 1520, + "max_properties": 1522, + "minimum": 0.764, + "maximum": 0.766, + "min_length": 1061, + "max_length": 1063, + "pattern": "pattern_value", + "example": {}, + "any_of": {}, + "additional_properties": {}, + "ref": "ref_value", + "defs": {}, + }, + "response_json_schema": {}, + "routing_config": { + "auto_mode": {"model_routing_preference": 1}, + "manual_mode": { + "model_name": "model_name_value" + }, + }, + "audio_timestamp": True, + "response_modalities": [1], + "media_resolution": 1, + "speech_config": { + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "voice_name_value" + }, + "replicated_voice_config": { + "mime_type": "mime_type_value", + "voice_sample_audio": b"voice_sample_audio_blob", + }, + }, + "language_code": "language_code_value", + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "speaker_value", + "voice_config": {}, + } + ] + }, + }, + "thinking_config": { + "include_thoughts": True, + "thinking_budget": 1590, + "thinking_level": 1, + }, + "image_config": { + "image_output_options": { + "mime_type": "mime_type_value", + "compression_quality": 2074, + }, + "aspect_ratio": "aspect_ratio_value", + "person_generation": 1, + "image_size": "image_size_value", + }, + }, + }, + "additional_config": {}, + }, + "pointwise_metric_spec": { + "metric_prompt_template": "metric_prompt_template_value", + "system_instruction": "system_instruction_value", + "custom_output_format_config": {"return_raw_output": True}, + }, + "pairwise_metric_spec": { + "metric_prompt_template": "metric_prompt_template_value", + "candidate_response_field_name": "candidate_response_field_name_value", + "baseline_response_field_name": "baseline_response_field_name_value", + "system_instruction": "system_instruction_value", + "custom_output_format_config": {}, + }, + "exact_match_spec": {}, + "bleu_spec": {"use_effective_order": True}, + "rouge_spec": { + "rouge_type": "rouge_type_value", + "use_stemmer": True, + "split_summaries": True, + }, + "aggregation_metrics": [1], + } + ], + "output_config": { + "gcs_destination": {"output_uri_prefix": "output_uri_prefix_value"} + }, + "autorater_config": {}, + "inference_generation_config": {}, + }, }, "name": "name_value", "tuned_model_display_name": "tuned_model_display_name_value", @@ -6444,7 +6788,7 @@ async def test_create_tuning_job_rest_asyncio_call_success(request_type): }, "function_call": { "name": "name_value", - "args": {"fields": {}}, + "args": {}, "partial_args": [ { "null_value": 0, @@ -6505,6 +6849,50 @@ async def test_create_tuning_job_rest_asyncio_call_success(request_type): }, "encryption_spec": {"kms_key_name": "kms_key_name_value"}, "service_account": "service_account_value", + "evaluate_dataset_runs": [ + { + "operation_name": "operation_name_value", + "evaluation_run": "evaluation_run_value", + "checkpoint_id": "checkpoint_id_value", + "evaluate_dataset_response": { + "aggregation_output": { + "dataset": { + "gcs_source": {"uris": ["uris_value1", "uris_value2"]}, + "bigquery_source": {"input_uri": "input_uri_value"}, + }, + "aggregation_results": [ + { + "pointwise_metric_result": { + "score": 0.54, + "explanation": "explanation_value", + "custom_output": { + "raw_outputs": { + "raw_output": [ + "raw_output_value1", + "raw_output_value2", + ] + } + }, + }, + "pairwise_metric_result": { + "pairwise_choice": 1, + "explanation": "explanation_value", + "custom_output": {}, + }, + "exact_match_metric_value": {"score": 0.54}, + "bleu_metric_value": {"score": 0.54}, + "rouge_metric_value": {"score": 0.54}, + "aggregation_metric": 1, + } + ], + }, + "output_info": { + "gcs_output_directory": "gcs_output_directory_value" + }, + }, + "error": {}, + } + ], } # The version of a generated dependency at test runtime may differ from the version used during generation. # Delete any fields which are not present in the current runtime dependency diff --git a/tests/unit/gapic/aiplatform_v1/test_migration_service.py b/tests/unit/gapic/aiplatform_v1/test_migration_service.py index 1aa93934c0..827bb18c1e 100644 --- a/tests/unit/gapic/aiplatform_v1/test_migration_service.py +++ b/tests/unit/gapic/aiplatform_v1/test_migration_service.py @@ -5623,19 +5623,22 @@ def test_parse_dataset_path(): def test_dataset_path(): project = "squid" - dataset = "clam" - expected = "projects/{project}/datasets/{dataset}".format( + location = "clam" + dataset = "whelk" + expected = "projects/{project}/locations/{location}/datasets/{dataset}".format( project=project, + location=location, dataset=dataset, ) - actual = MigrationServiceClient.dataset_path(project, dataset) + actual = MigrationServiceClient.dataset_path(project, location, dataset) assert expected == actual def test_parse_dataset_path(): expected = { - "project": "whelk", - "dataset": "octopus", + "project": "octopus", + "location": "oyster", + "dataset": "nudibranch", } path = MigrationServiceClient.dataset_path(**expected) @@ -5645,22 +5648,19 @@ def test_parse_dataset_path(): def test_dataset_path(): - project = "oyster" - location = "nudibranch" - dataset = "cuttlefish" - expected = "projects/{project}/locations/{location}/datasets/{dataset}".format( + project = "cuttlefish" + dataset = "mussel" + expected = "projects/{project}/datasets/{dataset}".format( project=project, - location=location, dataset=dataset, ) - actual = MigrationServiceClient.dataset_path(project, location, dataset) + actual = MigrationServiceClient.dataset_path(project, dataset) assert expected == actual def test_parse_dataset_path(): expected = { - "project": "mussel", - "location": "winkle", + "project": "winkle", "dataset": "nautilus", } path = MigrationServiceClient.dataset_path(**expected)