From 6f87b355ed0689265969dff8fc388760a4e5f718 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Tue, 25 Aug 2026 13:43:48 +0200 Subject: [PATCH] Make a preset workload state the requested dataset `random` was dstack's own name for a synthetic workload, but the report's `dataset` carried the benchmark tool's name for the data it served, so a shared-prefix benchmark run with SGLang could never verify (#4198): two namespaces, compared as one. Now the workload states the request. `dataset` echoes the requested dataset and is absent for a synthetic workload, whose requested shared prefix is compared instead - the one fact the request and the report share. The tool's own name for its generated data stays where it already was, in `command`. The stored record is normalized from the configuration, the authority on what was requested, so nothing the agent volunteers can contradict the contract - and the system prompt needs no change. `dataset: random` is retired from the configuration: a set dataset always means a real one, and synthetic prompts are requested by omitting it. Records it was legal in are upgraded when read - stored presets, in both the configuration and the workload, and an in-flight session's saved configuration. Based on dstackai/dstack#4199: the verification comparisons and much of the test suite are Victor's. Co-authored-by: Victor Skvortsov Co-Authored-By: Claude Fable 5 --- .../Presets/Details/Constraints/index.tsx | 12 +- .../_internal/cli/services/presets/create.py | 17 +- .../_internal/cli/services/presets/output.py | 11 +- .../_internal/cli/services/presets/store.py | 15 ++ .../_internal/cli/services/presets/verify.py | 63 ++++++- .../_internal/core/models/configurations.py | 22 ++- src/dstack/_internal/core/models/presets.py | 36 ++-- src/tests/_internal/cli/common.py | 14 ++ .../_internal/cli/models/test_presets.py | 23 ++- .../cli/services/presets/test_create.py | 5 +- .../cli/services/presets/test_store.py | 25 ++- .../cli/services/presets/test_verify.py | 167 +++++++++++++++++- .../core/models/test_configurations.py | 15 +- .../_internal/core/models/test_presets.py | 37 ++-- 14 files changed, 387 insertions(+), 75 deletions(-) diff --git a/frontend/src/pages/Presets/Details/Constraints/index.tsx b/frontend/src/pages/Presets/Details/Constraints/index.tsx index 5d01286e1..75450fea7 100644 --- a/frontend/src/pages/Presets/Details/Constraints/index.tsx +++ b/frontend/src/pages/Presets/Details/Constraints/index.tsx @@ -7,8 +7,6 @@ import { Box, ColumnLayout, Container, Header, Loader } from 'components'; import { formatTokenCount } from 'libs/presets'; import { useGetPresetQuery } from 'services/preset'; -const DEFAULT_DATASET = 'random'; - export const PresetConstraints: FC = () => { const { t } = useTranslation(); const params = useParams(); @@ -37,10 +35,12 @@ export const PresetConstraints: FC = () => { return ( {t('presets.constraints')}}> -
- {t('presets.dataset')} -
{dataset ?? DEFAULT_DATASET}
-
+ {dataset && ( +
+ {t('presets.dataset')} +
{dataset}
+
+ )}
{t('presets.input_tokens')}
{formatTokenCount(inputTokens)}
diff --git a/src/dstack/_internal/cli/services/presets/create.py b/src/dstack/_internal/cli/services/presets/create.py index f4157f722..d075ad199 100644 --- a/src/dstack/_internal/cli/services/presets/create.py +++ b/src/dstack/_internal/cli/services/presets/create.py @@ -74,7 +74,6 @@ from dstack._internal.cli.utils.offers import print_offers_table from dstack._internal.core.errors import CLIError, ConfigurationError from dstack._internal.core.models.configurations import ( - DEFAULT_DATASET, PresetConfiguration, TaskConfiguration, ) @@ -176,9 +175,12 @@ def load_session_configuration(session: PresetSession) -> PresetConfiguration: f" followed; run `dstack preset resume {session.preset_id}` instead" ) try: - return PresetConfiguration.model_validate( - yaml.safe_load(configuration_path.read_text(encoding="utf-8")) - ) + data = yaml.safe_load(configuration_path.read_text(encoding="utf-8")) + # A session started before the `random` alias was retired legally saved + # it; the rejection is for fresh configurations, not this record. + if isinstance(data, dict) and data.get("dataset") == "random": + data = {key: value for key, value in data.items() if key != "dataset"} + return PresetConfiguration.model_validate(data) except (OSError, ValueError) as e: raise CLIError(f"Could not read the preset configuration: {e}") from e @@ -615,7 +617,7 @@ async def _create_preset( user_prompt=setup.user_prompt, baseline=configuration.effective_baseline, previous=setup.previous, - custom_dataset=configuration.effective_dataset != DEFAULT_DATASET, + custom_dataset=configuration.dataset is not None, ) if setup.write_constraints: if setup.user_prompt: @@ -916,8 +918,7 @@ def _build_constraints( build_name: str, allowed_fleets: Sequence[str], ) -> str: - dataset = configuration.effective_dataset - if dataset == DEFAULT_DATASET: + if configuration.dataset is None: constraints: PresetConstraints = PresetRandomConstraints( run_name_prefix=build_name, model=configuration.model, @@ -940,7 +941,7 @@ def _build_constraints( max_ttft=configuration.max_ttft, trials_num=configuration.trials, concurrency=configuration.concurrency, - dataset=dataset, + dataset=configuration.dataset, baseline=configuration.effective_baseline, fleets=list(allowed_fleets), env=list(configuration.env), diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 0882d31bc..05db1828c 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -6,7 +6,7 @@ from dstack._internal.cli.models.presets import AnyStoredPreset, VerifiedPreset from dstack._internal.cli.utils.common import add_row_from_dict, console -from dstack._internal.core.models.configurations import DEFAULT_DATASET, PresetConfiguration +from dstack._internal.core.models.configurations import PresetConfiguration from dstack._internal.core.models.presets import PresetWorkload from dstack._internal.utils.common import pretty_date, pretty_resources @@ -294,15 +294,14 @@ def format_preset_objective( # A pulled preset carries no creation context; the measured workload is its # honest record of the conditions the numbers hold for. parts = [] - if workload.dataset != DEFAULT_DATASET: + if workload.dataset is not None: parts.append(f"data={workload.dataset}") else: parts.append( f"io={_format_token_count(workload.input_tokens)}" f"/{_format_token_count(workload.output_tokens)}" ) - shared_prefix_tokens = getattr(workload, "shared_prefix_tokens", 0) - share = round(100 * shared_prefix_tokens / workload.input_tokens) + share = round(100 * workload.shared_prefix_tokens / workload.input_tokens) if share: parts.append(f"prefix={share}%") parts.append(f"c={workload.concurrency}") @@ -313,8 +312,8 @@ def _format_creation_objective( configuration: PresetConfiguration, workload: PresetWorkload, *, verbose: bool ) -> str: parts = [] - if configuration.effective_dataset != DEFAULT_DATASET: - parts.append(f"data={configuration.effective_dataset}") + if configuration.dataset is not None: + parts.append(f"data={configuration.dataset}") else: input_tokens = configuration.effective_input_tokens parts.append( diff --git a/src/dstack/_internal/cli/services/presets/store.py b/src/dstack/_internal/cli/services/presets/store.py index 21f262b16..621168460 100644 --- a/src/dstack/_internal/cli/services/presets/store.py +++ b/src/dstack/_internal/cli/services/presets/store.py @@ -155,6 +155,7 @@ def _load(self, path: Path) -> AnyStoredPreset: upgraded = _upgrade_model_field(upgraded) upgraded = _upgrade_submitted_at(upgraded) upgraded = _upgrade_untagged_preset(upgraded) + upgraded = _upgrade_random_dataset_alias(upgraded) preset = _STORED_PRESET_ADAPTER.validate_python(upgraded) except (OSError, ValidationError, yaml.YAMLError) as e: if isinstance(data, dict) and "validations" in data: @@ -202,6 +203,20 @@ def _upgrade_untagged_preset(data: Any) -> Any: return {**data, "status": status} +# TODO: Remove in 0.22 +def _upgrade_random_dataset_alias(data: Any) -> Any: + """`random` used to be the explicit way to request synthetic prompts, and its + own validator suggested writing it; the retired alias is rejected in a fresh + configuration but cannot invalidate a record it was legal in.""" + configuration = data.get("configuration") if isinstance(data, dict) else None + if not isinstance(configuration, dict) or configuration.get("dataset") != "random": + return data + return { + **data, + "configuration": {key: value for key, value in configuration.items() if key != "dataset"}, + } + + # TODO: Remove in 0.22 def _upgrade_pre_0_21_2_preset(data: dict, *, preset_id: str) -> dict: """A preset file from before 0.21.2 stored the service as `service`, the diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 5ca56c670..36fcda5fd 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -27,7 +27,11 @@ from dstack._internal.core.errors import CLIError from dstack._internal.core.models.configurations import PresetConfiguration, ServiceConfiguration from dstack._internal.core.models.envs import EnvSentinel -from dstack._internal.core.models.presets import PresetVerificationReplicaGroup +from dstack._internal.core.models.presets import ( + PresetBenchmark, + PresetVerificationReplicaGroup, + PresetWorkload, +) from dstack._internal.core.models.runs import JobStatus, Run, RunStatus @@ -110,7 +114,7 @@ def build_verified_preset( # it cannot be confused with the service's client-facing model name. repo=report.model, context_length=report.context_length, - benchmark=report.benchmark, + benchmark=_normalized_benchmark(report.benchmark, preset_configuration), best_trial=report.trial, configuration=preset_configuration, preset_id=preset_id, @@ -134,11 +138,10 @@ def _verified_run_service(run: Run, report: PresetAgentSuccess) -> ServiceConfig def _check_report_answers_request( report: PresetAgentSuccess, configuration: PresetConfiguration ) -> None: - """The report must answer what the configuration asked: the same dataset, and - the requested model — exactly when it was exact, any variant of the base - otherwise.""" - if report.benchmark.workload.dataset != configuration.effective_dataset: - raise CLIError("Claude final benchmark dataset does not match the requested dataset") + """The report must answer what the configuration asked: the same benchmark + workload, and the requested model — exactly when it was exact, any variant of + the base otherwise.""" + _check_workload_answers_request(report.benchmark.workload, configuration) if configuration.model.allows_variant_selection: if report.base != configuration.model.api_model_name: raise CLIError("Claude final report base does not match the requested model") @@ -146,6 +149,52 @@ def _check_report_answers_request( raise CLIError("Claude changed an exact model request") +def _check_workload_answers_request( + workload: PresetWorkload, configuration: PresetConfiguration +) -> None: + """Only what the configuration specifies exactly is compared. A named dataset is + compared by name, because the request and the report both use the dataset's own + name. A synthetic workload has no such shared name — the report's `dataset`, if + any, is the benchmark tool's own name for the data it generated — so the shared + prefix it was run with is compared instead. `input_tokens` and `output_tokens` + are what the benchmark measured rather than an echo of the request, so they are + not compared.""" + if configuration.dataset is not None: + if workload.dataset != configuration.dataset: + raise CLIError( + f"Claude final benchmark dataset {workload.dataset!r} does not match the" + f" requested dataset {configuration.dataset!r}" + ) + else: + shared_prefix_tokens = configuration.shared_prefix_tokens or 0 + if workload.shared_prefix_tokens != shared_prefix_tokens: + raise CLIError( + f"Claude final benchmark shared prefix of {workload.shared_prefix_tokens}" + f" tokens does not match the requested {shared_prefix_tokens}" + ) + if configuration.concurrency is not None and workload.concurrency != configuration.concurrency: + raise CLIError( + f"Claude final benchmark concurrency of {workload.concurrency} does not match the" + f" requested concurrency of {configuration.concurrency}" + ) + + +def _normalized_benchmark( + benchmark: PresetBenchmark, configuration: PresetConfiguration +) -> PresetBenchmark: + """The stored workload states the request, and the configuration is the + authority on what was requested: a synthetic run carries no dataset — whatever + the benchmark tool called its generated data is already on record in `command` + — and a dataset run carries no shared prefix. The agent's report may volunteer + either; neither is trusted into the record.""" + workload = benchmark.workload.model_copy( + update=( + {"dataset": None} if configuration.dataset is None else {"shared_prefix_tokens": 0} + ) + ) + return benchmark.model_copy(update={"workload": workload}) + + def _portable_service( service: ServiceConfiguration, configuration: PresetConfiguration, diff --git a/src/dstack/_internal/core/models/configurations.py b/src/dstack/_internal/core/models/configurations.py index 4378849bc..344d12aca 100644 --- a/src/dstack/_internal/core/models/configurations.py +++ b/src/dstack/_internal/core/models/configurations.py @@ -1593,7 +1593,6 @@ def replica_groups(self) -> List[ReplicaGroup]: DEFAULT_INPUT_TOKENS = 1024 DEFAULT_OUTPUT_TOKENS = 1024 DEFAULT_BASELINE = True -DEFAULT_DATASET = "random" class PresetModelRepo(CoreModel): @@ -1794,10 +1793,9 @@ class PresetConfiguration( Optional[str], Field( description=( - "The benchmark dataset used during preset creation: `random` for synthetic" - " prompts shaped by `input_tokens` and `output_tokens`, a benchmark tool's" - " dataset name (e.g. `sharegpt`, `spec_bench`), or a Hugging Face dataset ID." - " Defaults to `random`" + "The benchmark dataset used during preset creation: a benchmark tool's" + " dataset name (e.g. `sharegpt`, `spec_bench`) or a Hugging Face dataset ID." + " Omit for synthetic prompts shaped by `input_tokens` and `output_tokens`" ) ), ] = None @@ -1838,10 +1836,6 @@ def effective_output_tokens(self) -> int: def effective_baseline(self) -> bool: return self.baseline if self.baseline is not None else DEFAULT_BASELINE - @property - def effective_dataset(self) -> str: - return self.dataset if self.dataset is not None else DEFAULT_DATASET - @field_validator("dataset") @classmethod def validate_dataset_name(cls, value: Optional[str]) -> Optional[str]: @@ -1852,11 +1846,15 @@ def validate_dataset_name(cls, value: Optional[str]) -> Optional[str]: value = value.strip() if not value: raise ValueError("dataset must be a non-empty string") + if value == "random": + # The retired alias for the default. A set dataset now always means a + # real one; synthetic prompts are requested by omitting it. + raise ValueError("`random` is not a dataset; omit `dataset` for synthetic prompts") return value @model_validator(mode="after") def validate_dataset(self) -> Self: - if self.dataset in (None, DEFAULT_DATASET): + if self.dataset is None: return self set_fields = [ name @@ -1865,8 +1863,8 @@ def validate_dataset(self) -> Self: ] if set_fields: raise ValueError( - f"{', '.join(set_fields)} can only be set with the `random` dataset;" - " a custom dataset defines its own request shape" + f"{', '.join(set_fields)} shape synthetic prompts and cannot be set" + " together with `dataset`; a dataset defines its own request shape" ) return self diff --git a/src/dstack/_internal/core/models/presets.py b/src/dstack/_internal/core/models/presets.py index ab37cc6c8..95e87fb29 100644 --- a/src/dstack/_internal/core/models/presets.py +++ b/src/dstack/_internal/core/models/presets.py @@ -1,5 +1,5 @@ import re -from typing import Any, List, Literal, Optional, Sequence, Union +from typing import Any, List, Literal, Optional, Sequence from pydantic import Field, PositiveFloat, PositiveInt, field_validator, model_validator from typing_extensions import Annotated, Self @@ -36,18 +36,34 @@ class PresetWorkload(CoreModel): # A Literal, not a str: the agent-facing JSON schema is generated from this # model, so the allowed values must be part of it. api: Literal["chat_completions", "completions"] - dataset: str + dataset: Optional[str] = None + """The dataset the configuration requested, which the benchmark served. + Absent for a synthetic workload: no dataset was requested, and the name the + benchmark tool gives the data it generates is its own affair, on record in + `command`.""" num_requests: PositiveInt input_tokens: PositiveInt output_tokens: Annotated[int, Field(ge=2)] concurrency: PositiveInt - - -class PresetRandomWorkload(PresetWorkload): - # A Literal, not a defaulted str: `PresetBenchmark.workload` is a union with - # the base model, and only a literal `dataset` discriminates it. - dataset: Literal["random"] = "random" shared_prefix_tokens: Annotated[int, Field(ge=0)] = 0 + """How many leading tokens every measured request shared. Only a synthetic + workload has one: a named dataset defines its own requests.""" + + @model_validator(mode="before") + @classmethod + def drop_legacy_random_dataset(cls, data: Any) -> Any: + # Before the dataset meant the requested one, every synthetic workload + # was stored as the literal `random` next to its shared prefix. The + # combination can mean nothing else: `random` is rejected as a + # configuration dataset, and a dataset defines its own requests, so a + # dataset workload never records a prefix. + if ( + isinstance(data, dict) + and data.get("dataset") == "random" + and "shared_prefix_tokens" in data + ): + data = {key: value for key, value in data.items() if key != "dataset"} + return data class PresetBenchmarkLatency(CoreModel): @@ -78,9 +94,7 @@ class PresetBenchmark(CoreModel): tool: Annotated[str, Field(min_length=1)] tool_version: Annotated[str, Field(min_length=1)] command: Annotated[str, Field(min_length=1)] - # The subclass first: a report without `dataset` is a random workload, and a - # base-typed field would reject its `shared_prefix_tokens` as unknown. - workload: Union[PresetRandomWorkload, PresetWorkload] + workload: PresetWorkload metrics: PresetBenchmarkMetrics @property diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index 86b15c666..35ae8bb78 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -79,6 +79,20 @@ def run_dstack_cli( return exit_code +# The trial workload from the SGLang session in dstackai/dstack#4198: a synthetic +# shared-prefix benchmark run with a tool that does not call its generated data +# `random`. It used to match neither of the two workload models the schema offered. +SHARED_PREFIX_WORKLOAD = { + "api": "completions", + "dataset": "generated-shared-prefix", + "num_requests": 16, + "input_tokens": 131072, + "output_tokens": 512, + "concurrency": 4, + "shared_prefix_tokens": 130048, +} + + def get_preset_benchmark() -> PresetBenchmark: benchmark = PresetBenchmark( tool="vllm bench serve", diff --git a/src/tests/_internal/cli/models/test_presets.py b/src/tests/_internal/cli/models/test_presets.py index ca3d05f54..a0df68fbd 100644 --- a/src/tests/_internal/cli/models/test_presets.py +++ b/src/tests/_internal/cli/models/test_presets.py @@ -4,7 +4,7 @@ from dstack._internal.core.models.presets import ( PresetBenchmark, ) -from tests._internal.cli.common import get_preset_benchmark +from tests._internal.cli.common import SHARED_PREFIX_WORKLOAD, get_preset_benchmark pytestmark = pytest.mark.windows @@ -31,3 +31,24 @@ def test_rejects_tool_specific_metrics(self): # permitted" rather than "extra fields not permitted"). What matters is the rejection. with pytest.raises(ValidationError): PresetBenchmark.model_validate(data) + + def test_keeps_both_a_tool_dataset_name_and_a_shared_prefix(self): + # A synthetic workload has two facts to state: the shared prefix it was run + # with and the name the benchmark tool gave the data it generated. + data = get_preset_benchmark().model_dump() + data["workload"] = dict(SHARED_PREFIX_WORKLOAD) + + benchmark = PresetBenchmark.model_validate(data) + + assert benchmark.workload.dataset == "generated-shared-prefix" + assert benchmark.workload.shared_prefix_tokens == 130048 + + def test_reads_a_workload_stored_without_a_dataset(self): + # Every preset written before the workload could carry a tool dataset name. + data = get_preset_benchmark().model_dump() + del data["workload"]["dataset"] + + benchmark = PresetBenchmark.model_validate(data) + + assert benchmark.workload.dataset is None + assert benchmark.workload.shared_prefix_tokens == 0 diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index ebf962014..e3ae148d8 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -99,7 +99,8 @@ def creation_context(tmp_path, monkeypatch): base="Qwen/Qwen3.5-27B", min_context_length=8192, max_ttft=5000, - concurrency=8, + # Matches the fixture report's benchmark concurrency, which verification compares. + concurrency=1, trials=1, fleets=["gpu-fleet"], env={"LICENSE": "license-secret", "TOKENIZERS_PARALLELISM": "false"}, @@ -109,7 +110,7 @@ def creation_context(tmp_path, monkeypatch): base="Qwen/Qwen3.5-27B", min_context_length=8192, max_ttft=5000, - concurrency=8, + concurrency=1, trials=1, fleets=["gpu-fleet"], env=["LICENSE", "TOKENIZERS_PARALLELISM=false"], diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index 28bca01f5..a2895aa79 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -74,6 +74,28 @@ def test_loads_a_preset_written_by_0_21(self, tmp_path: Path): assert loaded == preset + def test_loads_a_preset_that_requested_the_random_dataset(self, tmp_path: Path): + # A synthetic preset the released version wrote with an explicit + # `dataset: random` — the alias its own validator used to suggest. The + # retired alias is rejected in a fresh configuration, but cannot + # invalidate a record it was legal in: it reads back as what it meant, + # a synthetic workload with no requested dataset. + store = PresetStore(tmp_path / "presets") + preset = get_preset() + store.save(preset) + path = tmp_path / "presets" / preset.id / "preset.yml" + data = yaml.safe_load(path.read_text()) + data["configuration"]["dataset"] = "random" + data["benchmark"]["workload"]["dataset"] = "random" + data["benchmark"]["workload"]["shared_prefix_tokens"] = 0 + path.write_text(yaml.safe_dump(data, sort_keys=False)) + + loaded = store.get(preset.id) + + assert loaded is not None + assert loaded.configuration.dataset is None + assert loaded.benchmark.workload.dataset is None + def test_loads_a_preset_that_predates_the_repo_field(self, tmp_path: Path): # Presets written before the rename store the served repo as `model`. store = PresetStore(tmp_path / "presets") @@ -341,7 +363,8 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): group.name for group in preset.service.replica_groups ] assert preset.verified_on[0].replicas[0].gpu.name == ["MI300X"] - assert preset.benchmark.workload.dataset == "random" + # The old format never recorded a dataset name for a synthetic workload. + assert preset.benchmark.workload.dataset is None assert store.list() == [preset] def test_upgrade_maps_validation_replicas_to_replica_groups(self, tmp_path: Path): diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index 014d32d58..26750c7b7 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -1,9 +1,10 @@ from datetime import datetime, timezone +from typing import Any import pytest from pydantic import ValidationError -from dstack._internal.cli.models.preset_agent import AnyPresetAgentResult +from dstack._internal.cli.models.preset_agent import AnyPresetAgentResult, PresetAgentSuccess from dstack._internal.cli.services.presets.agent import ( PresetAgentProcessOutput, ) @@ -20,8 +21,11 @@ from dstack._internal.core.models.configurations import PresetConfiguration from dstack._internal.core.models.envs import EnvSentinel from dstack._internal.core.models.files import FilePathMapping +from dstack._internal.core.models.presets import PresetWorkload from dstack._internal.core.models.profiles import ProfileParams +from dstack._internal.core.models.runs import Run from tests._internal.cli.common import ( + SHARED_PREFIX_WORKLOAD, get_preset, get_preset_benchmark, get_running_service_run, @@ -170,12 +174,121 @@ def test_rejects_a_file_without_a_mirrored_copy(self, tmp_path): created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - def test_rejects_benchmark_on_a_different_dataset(self, tmp_path): + def test_verifies_a_shared_prefix_benchmark_named_by_the_benchmark_tool(self, tmp_path): + # `random` is dstack's own name for a synthetic workload; the report carries + # the benchmark tool's name for the data it generated, which is + # `generated-shared-prefix` for SGLang and `random` only for vLLM. Comparing + # the two rejected a benchmark that answered the request exactly. run = get_running_service_run() - # The report's workload defaults to `random`, but the configuration - # demanded a custom dataset: the benchmark does not match the contract. - with pytest.raises(CLIError, match="dataset does not match"): + preset = build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(), + report=_shared_prefix_report(run), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + # The stored workload states the request: no dataset was asked for, so + # none is recorded — the tool's own name for its generated data stays in + # `command` — while the requested prefix survives into the record. + assert preset.benchmark.workload.dataset is None + assert preset.benchmark.workload.shared_prefix_tokens == 130048 + + def test_rejects_a_benchmark_without_the_requested_shared_prefix(self, tmp_path): + run = get_running_service_run() + + with pytest.raises( + CLIError, match="shared prefix of 0 tokens does not match the requested 130048" + ): + build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(), + report=_shared_prefix_report(run, shared_prefix_tokens=0), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): + run = get_running_service_run() + + with pytest.raises( + CLIError, match="concurrency of 8 does not match the requested concurrency of 4" + ): + build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(), + report=_shared_prefix_report(run, concurrency=8), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): + run = get_running_service_run() + report = _dataset_report(run, dataset="spec_bench") + + preset = build_verified_preset( + run=run, + preset_configuration=PresetConfiguration( + name="qwen-build", + base="Qwen/Qwen3.5-27B", + dataset="spec_bench", + ), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + assert preset.benchmark.workload.dataset == "spec_bench" + + def test_stores_a_dataset_benchmark_without_a_stray_shared_prefix(self, tmp_path): + # The mirror of the synthetic case: a dataset defines its own requests, + # so a prefix the report volunteers does not enter the record. + run = get_running_service_run() + + preset = build_verified_preset( + run=run, + preset_configuration=PresetConfiguration( + name="qwen-build", + base="Qwen/Qwen3.5-27B", + min_context_length=8192, + gateway="benchmark-gateway", + dataset="spec_bench", + ), + report=_dataset_report(run, dataset="spec_bench", shared_prefix_tokens=768), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + created_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + assert preset.benchmark.workload.dataset == "spec_bench" + assert preset.benchmark.workload.shared_prefix_tokens == 0 + + # A named dataset is the one dataset name the report and the request share, so + # the report either echoes it or does not answer the request. + @pytest.mark.parametrize("reported", ["sharegpt", None]) + def test_rejects_benchmark_on_a_different_dataset(self, tmp_path, reported): + run = get_running_service_run() + + # Both values are named: "does not match" alone leaves nothing to act on. + with pytest.raises( + CLIError, + match=f"dataset {reported!r} does not match the requested dataset 'spec_bench'", + ): build_verified_preset( run=run, preset_configuration=PresetConfiguration( @@ -183,7 +296,7 @@ def test_rejects_benchmark_on_a_different_dataset(self, tmp_path): base="Qwen/Qwen3.5-27B", dataset="spec_bench", ), - report=get_successful_preset_report(run), + report=_dataset_report(run, dataset=reported), workspace_path=tmp_path, session_path=tmp_path, preset_id="ab12cd34", @@ -262,3 +375,45 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): assert report.benchmark is not None assert "bearer header" in report.benchmark.command + + +def _shared_prefix_configuration() -> PresetConfiguration: + """The configuration from dstackai/dstack#4198: a shared prefix and no dataset.""" + return PresetConfiguration.model_validate( + { + "type": "preset", + "base": "Qwen/Qwen3.5-27B", + "min_context_length": 262144, + "max_ttft": 5000, + "trials": 4, + "concurrency": 4, + "input_tokens": 131072, + "output_tokens": 512, + "shared_prefix_tokens": 130048, + } + ) + + +def _shared_prefix_report(run: Run, **workload: Any) -> PresetAgentSuccess: + return _report_with_workload(run, {**SHARED_PREFIX_WORKLOAD, **workload}) + + +def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: + # A dataset defines the request shape, so the workload records what it measured. + return _report_with_workload( + run, + { + "api": "chat_completions", + "num_requests": 16, + "input_tokens": 347, + "output_tokens": 2451, + "concurrency": 4, + **workload, + }, + ) + + +def _report_with_workload(run: Run, workload: dict[str, Any]) -> PresetAgentSuccess: + benchmark = get_preset_benchmark() + benchmark.workload = PresetWorkload.model_validate(workload) + return get_successful_preset_report(run).model_copy(update={"benchmark": benchmark}) diff --git a/src/tests/_internal/core/models/test_configurations.py b/src/tests/_internal/core/models/test_configurations.py index d80590be8..e0d91c718 100644 --- a/src/tests/_internal/core/models/test_configurations.py +++ b/src/tests/_internal/core/models/test_configurations.py @@ -1311,22 +1311,27 @@ def test_requires_model(self): @pytest.mark.parametrize("field", ["input_tokens", "output_tokens", "shared_prefix_tokens"]) def test_rejects_request_shape_fields_with_a_custom_dataset(self, field): - with pytest.raises(ValidationError, match="only be set with the `random` dataset"): + with pytest.raises(ValidationError, match="cannot be set together with `dataset`"): PresetConfiguration(base="Qwen/Qwen3.5-27B", dataset="spec_bench", **{field: 512}) - def test_allows_request_shape_fields_with_the_random_dataset(self): + def test_allows_request_shape_fields_without_a_dataset(self): configuration = PresetConfiguration( - base="Qwen/Qwen3.5-27B", dataset="random", input_tokens=1024, output_tokens=256 + base="Qwen/Qwen3.5-27B", input_tokens=1024, output_tokens=256 ) assert configuration.input_tokens == 1024 assert configuration.output_tokens == 256 - def test_defaults_to_the_random_dataset(self): + def test_rejects_the_retired_random_alias(self): + # `random` used to be the explicit way to ask for synthetic prompts; + # now a set dataset always means a real one. + with pytest.raises(ValidationError, match="omit `dataset` for synthetic prompts"): + PresetConfiguration(base="Qwen/Qwen3.5-27B", dataset="random") + + def test_defaults_to_a_synthetic_workload(self): configuration = PresetConfiguration(base="Qwen/Qwen3.5-27B") assert configuration.dataset is None - assert configuration.effective_dataset == "random" class TestPresetConfigurationSchema: diff --git a/src/tests/_internal/core/models/test_presets.py b/src/tests/_internal/core/models/test_presets.py index edd72cbae..e12a88166 100644 --- a/src/tests/_internal/core/models/test_presets.py +++ b/src/tests/_internal/core/models/test_presets.py @@ -5,8 +5,6 @@ from dstack._internal.core.models.presets import ( PresetBenchmark, - PresetRandomWorkload, - PresetWorkload, validate_preset_file_path, validate_preset_file_paths, ) @@ -109,9 +107,9 @@ def test_applies_every_rule(self, paths: List[str], error: Optional[str]): class TestPresetBenchmarkWorkload: - """`api` and `dataset` are Literals, not plain strings: the agent-facing JSON - schema is generated from these models, and `dataset` is what discriminates - the workload union.""" + """`api` is a Literal, not a plain string: the agent-facing JSON schema is + generated from these models. `dataset` states the requested dataset, and its + absence states that none was — a synthetic workload.""" def test_rejects_an_unsupported_api(self): data = get_benchmark_data(get_workload_data(api="embeddings")) @@ -119,19 +117,38 @@ def test_rejects_an_unsupported_api(self): with pytest.raises(ValidationError): PresetBenchmark.model_validate(data) - def test_parses_a_dataset_workload_as_the_base_type(self): + def test_parses_a_dataset_workload(self): data = get_benchmark_data(get_workload_data(dataset="sharegpt")) benchmark = PresetBenchmark.model_validate(data) - assert type(benchmark.workload) is PresetWorkload assert benchmark.workload.dataset == "sharegpt" - def test_parses_a_workload_without_a_dataset_as_random(self): + def test_parses_a_workload_without_a_dataset_as_synthetic(self): data = get_benchmark_data(get_workload_data()) benchmark = PresetBenchmark.model_validate(data) - assert type(benchmark.workload) is PresetRandomWorkload - assert benchmark.workload.dataset == "random" + assert benchmark.workload.dataset is None assert benchmark.workload.shared_prefix_tokens == 0 + + def test_upgrades_a_stored_synthetic_workload(self): + # Synthetic workloads used to be stored as the literal `random` next to + # their shared prefix. The combination can mean nothing else: `random` is + # rejected as a configuration dataset, and a dataset workload never + # records a prefix. + data = get_benchmark_data(get_workload_data(dataset="random", shared_prefix_tokens=768)) + + benchmark = PresetBenchmark.model_validate(data) + + assert benchmark.workload.dataset is None + assert benchmark.workload.shared_prefix_tokens == 768 + + def test_keeps_a_dataset_named_random_without_a_prefix(self): + # Only the legacy combination is rewritten: a dataset workload never + # carries the prefix key, so one that names `random` stays as written. + data = get_benchmark_data(get_workload_data(dataset="random")) + + benchmark = PresetBenchmark.model_validate(data) + + assert benchmark.workload.dataset == "random"