Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions agentplatform/_genai/types/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -1874,6 +1874,11 @@ class Metric(_common.BaseModel):
description="""The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}""",
)

result_parsing_function: Optional[str] = Field(
default=None,
description="""Optional. A Python function string used to parse the raw output of the LLM judge model. The function must be named `parse_results` and accept a list of model response strings. It should return a dictionary with `score` (float) and `explanation` (str) keys. When provided, this is sent to the evaluation service as `result_parser_config.custom_code_parser_config.parsing_function`.""",
)

# Allow extra fields to support metric-specific config fields.
model_config = ConfigDict(extra="allow")

Expand Down Expand Up @@ -1959,6 +1964,11 @@ class LLMMetric(Metric):
description="""Optional. The name of the column in the EvaluationDataset containing the list of rubrics to use for this metric.""",
)

result_parsing_function: Optional[str] = Field(
default=None,
description="""Optional. A Python function string used to parse the raw output of the LLM judge model. The function must be named `parse_results` and accept a list of model response strings. It should return a dictionary with `score` (float) and `explanation` (str) keys. When provided, this is sent to the evaluation service as `result_parser_config.custom_code_parser_config.parsing_function`.""",
)

@field_validator("prompt_template", mode="before")
@classmethod
def validate_prompt_template(cls, value: Union[str, "MetricPromptBuilder"]) -> str:
Expand Down Expand Up @@ -2100,6 +2110,9 @@ class MetricDict(TypedDict, total=False):
metric_resource_name: Optional[str]
"""The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""

result_parsing_function: Optional[str]
"""Optional. A Python function string used to parse the raw output of the LLM judge model. The function must be named `parse_results` and accept a list of model response strings. It should return a dictionary with `score` (float) and `explanation` (str) keys. When provided, this is sent to the evaluation service as `result_parser_config.custom_code_parser_config.parsing_function`."""


MetricOrDict = Union[Metric, MetricDict]

Expand Down
8 changes: 8 additions & 0 deletions vertexai/_genai/types/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -1866,6 +1866,11 @@ class LLMMetric(Metric):
description="""Optional. The name of the column in the EvaluationDataset containing the list of rubrics to use for this metric.""",
)

result_parsing_function: Optional[str] = Field(
default=None,
description="""Optional. A Python function string used to parse the raw output of the LLM judge model. The function must be named `parse_results` and accept a list of model response strings. It should return a dictionary with `score` (float) and `explanation` (str) keys.""",
)

@field_validator("prompt_template", mode="before")
@classmethod
def validate_prompt_template(cls, value: Union[str, "MetricPromptBuilder"]) -> str:
Expand Down Expand Up @@ -2007,6 +2012,9 @@ class MetricDict(TypedDict, total=False):
metric_resource_name: Optional[str]
"""The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""

result_parsing_function: Optional[str]
"""Optional. A Python function string used to parse the raw output of the LLM judge model. The function must be named `parse_results` and accept a list of model response strings. It should return a dictionary with `score` (float) and `explanation` (str) keys."""


MetricOrDict = Union[Metric, MetricDict]

Expand Down
Loading