diff --git a/src/dstack/_internal/cli/models/presets.py b/src/dstack/_internal/cli/models/presets.py index c5ff005d2..93b874315 100644 --- a/src/dstack/_internal/cli/models/presets.py +++ b/src/dstack/_internal/cli/models/presets.py @@ -1,6 +1,6 @@ import re from datetime import datetime -from typing import Annotated, Literal, Optional, Union +from typing import Annotated, Literal, Optional from pydantic import ( Field, @@ -24,16 +24,19 @@ class PresetWorkload(CoreModel): api: Literal["chat_completions", "completions"] - dataset: str + dataset: Optional[str] = None + """The benchmark tool's own name for the data it served. It is the requested + dataset when the configuration named one; for the synthetic `random` workload it + is whatever the tool calls the data it generates (`random`, + `generated-shared-prefix`, ...), recorded but never compared with dstack's own + `random`.""" num_requests: PositiveInt input_tokens: PositiveInt output_tokens: Annotated[int, Field(ge=2)] concurrency: PositiveInt - - -class PresetRandomWorkload(PresetWorkload): - dataset: Literal["random"] = "random" shared_prefix_tokens: Annotated[int, Field(ge=0)] = 0 + """How many leading tokens every measured request shared. Only a synthetic + workload has one: a named dataset defines its own requests.""" class PresetBenchmarkLatency(CoreModel): @@ -64,9 +67,7 @@ class PresetBenchmark(CoreModel): tool: Annotated[str, Field(min_length=1)] tool_version: Annotated[str, Field(min_length=1)] command: Annotated[str, Field(min_length=1)] - # The subclass first: a report without `dataset` is a random workload, and a - # base-typed field would reject its `shared_prefix_tokens` as unknown. - workload: Union[PresetRandomWorkload, PresetWorkload] + workload: PresetWorkload metrics: PresetBenchmarkMetrics @property diff --git a/src/dstack/_internal/cli/services/presets/create.py b/src/dstack/_internal/cli/services/presets/create.py index 02fbfd92b..fca6cd371 100644 --- a/src/dstack/_internal/cli/services/presets/create.py +++ b/src/dstack/_internal/cli/services/presets/create.py @@ -77,7 +77,6 @@ from dstack._internal.core.models.envs import Env, EnvSentinel from dstack._internal.core.models.fleets import FleetStatus from dstack._internal.core.models.presets import ( - DEFAULT_DATASET, PresetConfiguration, PresetConstraints, PresetDatasetConstraints, @@ -613,7 +612,7 @@ async def _create_preset( user_prompt=setup.user_prompt, baseline=configuration.effective_baseline, previous=setup.previous, - custom_dataset=configuration.effective_dataset != DEFAULT_DATASET, + custom_dataset=configuration.has_custom_dataset, ) if setup.write_constraints: if setup.user_prompt: @@ -914,8 +913,7 @@ def _build_constraints( build_name: str, allowed_fleets: Sequence[str], ) -> str: - dataset = configuration.effective_dataset - if dataset == DEFAULT_DATASET: + if not configuration.has_custom_dataset: constraints: PresetConstraints = PresetRandomConstraints( run_name_prefix=build_name, model=configuration.model, @@ -938,7 +936,7 @@ def _build_constraints( max_ttft=configuration.max_ttft, trials_num=configuration.trials, concurrency=configuration.concurrency, - dataset=dataset, + dataset=configuration.effective_dataset, baseline=configuration.effective_baseline, fleets=list(allowed_fleets), env=list(configuration.env), diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 1d86ba5e4..c4eedb6bb 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -8,7 +8,6 @@ VerifiedPreset, ) from dstack._internal.cli.utils.common import add_row_from_dict, console -from dstack._internal.core.models.presets import DEFAULT_DATASET from dstack._internal.utils.common import pretty_date, pretty_resources _STATUS_DISPLAY = { @@ -291,7 +290,7 @@ def format_preset_objective( configuration = preset.configuration workload = preset.benchmark.workload parts = [] - if configuration.effective_dataset != DEFAULT_DATASET: + if configuration.has_custom_dataset: parts.append(f"data={configuration.effective_dataset}") else: input_tokens = configuration.effective_input_tokens diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index 0ebc1e92b..c3fcddd82 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -361,7 +361,7 @@ trial benchmarks in `trials//trial.json`, the final benchmark as "tool_version": "0.11.0", "command": "vllm bench serve ...", "workload": {"api": "chat_completions", "num_requests": 16, "input_tokens": 1024, "output_tokens": 128, "concurrency": 8, "dataset": "sharegpt"}, - "workload": {"api": "chat_completions", "num_requests": 16, "input_tokens": 1024, "output_tokens": 128, "concurrency": 8, "shared_prefix_tokens": 768}, + "workload": {"api": "chat_completions", "num_requests": 16, "input_tokens": 1024, "output_tokens": 128, "concurrency": 8, "shared_prefix_tokens": 768, "dataset": "random"}, "metrics": { "successful_requests": 16, "failed_requests": 0, "duration_seconds": 4.0, "total_input_tokens": 16384, "total_output_tokens": 2048, @@ -376,6 +376,11 @@ trial benchmarks in `trials//trial.json`, the final benchmark as Set `workload.dataset` to `dataset` from `constraints.json`, and compute `workload.input_tokens` and `workload.output_tokens` as the measured mean input and output token counts of the benchmark, rounded to whole tokens. + +Set `workload.shared_prefix_tokens` to `shared_prefix_tokens` from +`constraints.json`, and `workload.dataset` to the name the benchmark tool gives +the data it generated, whatever that name is. It records what ran and is never +required to be a particular value. Compute `output_tok_per_s` as `total_output_tokens / duration_seconds` and `per_user_tok_per_s` as `output_tok_per_s / workload.concurrency`. These are diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 3a763577b..b87912c67 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -11,6 +11,7 @@ ) from dstack._internal.cli.models.presets import ( PresetVerificationReplicaGroup, + PresetWorkload, VerifiedPreset, ) from dstack._internal.cli.services.presets.agent import ( @@ -134,11 +135,10 @@ def _verified_run_service(run: Run, report: PresetAgentSuccess) -> ServiceConfig def _check_report_answers_request( report: PresetAgentSuccess, configuration: PresetConfiguration ) -> None: - """The report must answer what the configuration asked: the same dataset, and - the requested model — exactly when it was exact, any variant of the base - otherwise.""" - if report.benchmark.workload.dataset != configuration.effective_dataset: - raise CLIError("Claude final benchmark dataset does not match the requested dataset") + """The report must answer what the configuration asked: the same benchmark + workload, and the requested model — exactly when it was exact, any variant of + the base otherwise.""" + _check_workload_answers_request(report.benchmark.workload, configuration) if configuration.model.allows_variant_selection: if report.base != configuration.model.api_model_name: raise CLIError("Claude final report base does not match the requested model") @@ -146,6 +146,36 @@ def _check_report_answers_request( raise CLIError("Claude changed an exact model request") +def _check_workload_answers_request( + workload: PresetWorkload, configuration: PresetConfiguration +) -> None: + """Only what the configuration specifies exactly is compared. A named dataset is + compared by name, because the request and the report both use the dataset's own + name. A synthetic workload has no such shared name: `random` is dstack's name for + it, while the report carries the benchmark tool's, so the shared prefix it was run + with is compared instead. `input_tokens` and `output_tokens` are what the + benchmark measured rather than an echo of the request, so they are not + compared.""" + if configuration.has_custom_dataset: + if workload.dataset != configuration.effective_dataset: + raise CLIError( + f"Claude final benchmark dataset {workload.dataset!r} does not match the" + f" requested dataset {configuration.effective_dataset!r}" + ) + else: + shared_prefix_tokens = configuration.shared_prefix_tokens or 0 + if workload.shared_prefix_tokens != shared_prefix_tokens: + raise CLIError( + f"Claude final benchmark shared prefix of {workload.shared_prefix_tokens}" + f" tokens does not match the requested {shared_prefix_tokens}" + ) + if configuration.concurrency is not None and workload.concurrency != configuration.concurrency: + raise CLIError( + f"Claude final benchmark concurrency of {workload.concurrency} does not match the" + f" requested concurrency of {configuration.concurrency}" + ) + + def _portable_service( service: ServiceConfiguration, configuration: PresetConfiguration, diff --git a/src/dstack/_internal/core/models/presets.py b/src/dstack/_internal/core/models/presets.py index 53cbed34e..f212780fe 100644 --- a/src/dstack/_internal/core/models/presets.py +++ b/src/dstack/_internal/core/models/presets.py @@ -268,6 +268,14 @@ def effective_baseline(self) -> bool: def effective_dataset(self) -> str: return self.dataset if self.dataset is not None else DEFAULT_DATASET + @property + def has_custom_dataset(self) -> bool: + """Whether a named dataset provides the benchmark requests. The default + `random` is not one: it is dstack's own name for synthetic prompts shaped by + `input_tokens` and `output_tokens`, and every benchmark tool has its own name + for the data it generates.""" + return self.effective_dataset != DEFAULT_DATASET + @field_validator("dataset") @classmethod def validate_dataset_name(cls, value: Optional[str]) -> Optional[str]: diff --git a/src/tests/_internal/cli/common.py b/src/tests/_internal/cli/common.py index 182c8d6a2..f94a11923 100644 --- a/src/tests/_internal/cli/common.py +++ b/src/tests/_internal/cli/common.py @@ -79,6 +79,20 @@ def run_dstack_cli( return exit_code +# The trial workload from the SGLang session in dstackai/dstack#4198: a synthetic +# shared-prefix benchmark run with a tool that does not call its generated data +# `random`. It used to match neither of the two workload models the schema offered. +SHARED_PREFIX_WORKLOAD = { + "api": "completions", + "dataset": "generated-shared-prefix", + "num_requests": 16, + "input_tokens": 131072, + "output_tokens": 512, + "concurrency": 4, + "shared_prefix_tokens": 130048, +} + + def get_preset_benchmark() -> PresetBenchmark: benchmark = PresetBenchmark( tool="vllm bench serve", diff --git a/src/tests/_internal/cli/models/test_presets.py b/src/tests/_internal/cli/models/test_presets.py index 5564aabe8..cf2c7589b 100644 --- a/src/tests/_internal/cli/models/test_presets.py +++ b/src/tests/_internal/cli/models/test_presets.py @@ -4,7 +4,7 @@ from dstack._internal.cli.models.presets import ( PresetBenchmark, ) -from tests._internal.cli.common import get_preset_benchmark +from tests._internal.cli.common import SHARED_PREFIX_WORKLOAD, get_preset_benchmark pytestmark = pytest.mark.windows @@ -31,3 +31,24 @@ def test_rejects_tool_specific_metrics(self): # permitted" rather than "extra fields not permitted"). What matters is the rejection. with pytest.raises(ValidationError): PresetBenchmark.model_validate(data) + + def test_keeps_both_a_tool_dataset_name_and_a_shared_prefix(self): + # A synthetic workload has two facts to state: the shared prefix it was run + # with and the name the benchmark tool gave the data it generated. + data = get_preset_benchmark().model_dump() + data["workload"] = dict(SHARED_PREFIX_WORKLOAD) + + benchmark = PresetBenchmark.model_validate(data) + + assert benchmark.workload.dataset == "generated-shared-prefix" + assert benchmark.workload.shared_prefix_tokens == 130048 + + def test_reads_a_workload_stored_without_a_dataset(self): + # Every preset written before the workload could carry a tool dataset name. + data = get_preset_benchmark().model_dump() + del data["workload"]["dataset"] + + benchmark = PresetBenchmark.model_validate(data) + + assert benchmark.workload.dataset is None + assert benchmark.workload.shared_prefix_tokens == 0 diff --git a/src/tests/_internal/cli/services/presets/test_create.py b/src/tests/_internal/cli/services/presets/test_create.py index 683fb55a9..370016c6a 100644 --- a/src/tests/_internal/cli/services/presets/test_create.py +++ b/src/tests/_internal/cli/services/presets/test_create.py @@ -99,7 +99,8 @@ def creation_context(tmp_path, monkeypatch): base="Qwen/Qwen3.5-27B", min_context_length=8192, max_ttft=5000, - concurrency=8, + # Matches the fixture report's benchmark concurrency, which verification compares. + concurrency=1, trials=1, fleets=["gpu-fleet"], env={"LICENSE": "license-secret", "TOKENIZERS_PARALLELISM": "false"}, @@ -109,7 +110,7 @@ def creation_context(tmp_path, monkeypatch): base="Qwen/Qwen3.5-27B", min_context_length=8192, max_ttft=5000, - concurrency=8, + concurrency=1, trials=1, fleets=["gpu-fleet"], env=["LICENSE", "TOKENIZERS_PARALLELISM=false"], diff --git a/src/tests/_internal/cli/services/presets/test_prompt.py b/src/tests/_internal/cli/services/presets/test_prompt.py index 1c163e627..8b3ae8217 100644 --- a/src/tests/_internal/cli/services/presets/test_prompt.py +++ b/src/tests/_internal/cli/services/presets/test_prompt.py @@ -59,6 +59,16 @@ def test_a_random_dataset_session_never_hears_of_datasets(self): assert "shared_prefix_tokens" in text assert "`dataset`" not in text + def test_asks_a_random_dataset_session_for_the_prefix_and_the_tool_s_name(self): + text = get_preset_agent_system_prompt( + user_prompt=None, baseline=False, previous=(), custom_dataset=False + ) + + # Both facts fit the benchmark schema, so the prompt asks for both: the + # prefix the request fixed, and whatever the tool calls the data it made. + assert "Set `workload.shared_prefix_tokens` to `shared_prefix_tokens`" in text + assert "`workload.dataset` to the name the benchmark tool gives" in text + def test_fails_loudly_when_the_prompt_has_no_directives(self, tmp_path, monkeypatch): plain = tmp_path / "system_prompt.md" plain.write_text("# Objective\n\nA prompt without directives.\n") diff --git a/src/tests/_internal/cli/services/presets/test_store.py b/src/tests/_internal/cli/services/presets/test_store.py index 0ed4b2cdf..765115f70 100644 --- a/src/tests/_internal/cli/services/presets/test_store.py +++ b/src/tests/_internal/cli/services/presets/test_store.py @@ -187,7 +187,8 @@ def test_upgrades_pre_0_21_2_preset(self, tmp_path: Path): group.name for group in preset.service.replica_groups ] assert preset.verified_on[0].replicas[0].gpu.name == ["MI300X"] - assert preset.benchmark.workload.dataset == "random" + # The old format never recorded a dataset name for a synthetic workload. + assert preset.benchmark.workload.dataset is None assert store.list() == [preset] def test_upgrade_maps_validation_replicas_to_replica_groups(self, tmp_path: Path): diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index 018254351..fdce8d7d6 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -1,9 +1,11 @@ from datetime import datetime, timezone +from typing import Any import pytest from pydantic import ValidationError -from dstack._internal.cli.models.preset_agent import AnyPresetAgentResult +from dstack._internal.cli.models.preset_agent import AnyPresetAgentResult, PresetAgentSuccess +from dstack._internal.cli.models.presets import PresetWorkload from dstack._internal.cli.services.presets.agent import ( PresetAgentProcessOutput, ) @@ -21,7 +23,9 @@ from dstack._internal.core.models.files import FilePathMapping from dstack._internal.core.models.presets import PresetConfiguration from dstack._internal.core.models.profiles import ProfileParams +from dstack._internal.core.models.runs import Run from tests._internal.cli.common import ( + SHARED_PREFIX_WORKLOAD, get_preset, get_preset_benchmark, get_running_service_run, @@ -170,12 +174,94 @@ def test_rejects_a_file_without_a_mirrored_copy(self, tmp_path): submitted_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), ) - def test_rejects_benchmark_on_a_different_dataset(self, tmp_path): + def test_verifies_a_shared_prefix_benchmark_named_by_the_benchmark_tool(self, tmp_path): + # `random` is dstack's own name for a synthetic workload; the report carries + # the benchmark tool's name for the data it generated, which is + # `generated-shared-prefix` for SGLang and `random` only for vLLM. Comparing + # the two rejected a benchmark that answered the request exactly. run = get_running_service_run() - # The report's workload defaults to `random`, but the configuration - # demanded a custom dataset: the benchmark does not match the contract. - with pytest.raises(CLIError, match="dataset does not match"): + preset = build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(), + report=_shared_prefix_report(run), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + submitted_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + assert preset.benchmark.workload.dataset == "generated-shared-prefix" + # The prefix the request asked for survives into the stored record. + assert preset.benchmark.workload.shared_prefix_tokens == 130048 + + def test_rejects_a_benchmark_without_the_requested_shared_prefix(self, tmp_path): + run = get_running_service_run() + + with pytest.raises( + CLIError, match="shared prefix of 0 tokens does not match the requested 130048" + ): + build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(), + report=_shared_prefix_report(run, shared_prefix_tokens=0), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + submitted_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_rejects_a_benchmark_at_another_concurrency(self, tmp_path): + run = get_running_service_run() + + with pytest.raises( + CLIError, match="concurrency of 8 does not match the requested concurrency of 4" + ): + build_verified_preset( + run=run, + preset_configuration=_shared_prefix_configuration(), + report=_shared_prefix_report(run, concurrency=8), + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + submitted_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + def test_verifies_a_benchmark_on_the_requested_dataset(self, tmp_path): + run = get_running_service_run() + report = _dataset_report(run, dataset="spec_bench") + + preset = build_verified_preset( + run=run, + preset_configuration=PresetConfiguration( + name="qwen-build", + base="Qwen/Qwen3.5-27B", + dataset="spec_bench", + ), + report=report, + workspace_path=tmp_path, + session_path=tmp_path, + preset_id="ab12cd34", + name=None, + submitted_at=datetime(2026, 1, 2, 3, 4, tzinfo=timezone.utc), + ) + + assert preset.benchmark.workload.dataset == "spec_bench" + + # A named dataset is the one dataset name the report and the request share, so + # the report either echoes it or does not answer the request. + @pytest.mark.parametrize("reported", ["sharegpt", None]) + def test_rejects_benchmark_on_a_different_dataset(self, tmp_path, reported): + run = get_running_service_run() + + # Both values are named: "does not match" alone leaves nothing to act on. + with pytest.raises( + CLIError, + match=f"dataset {reported!r} does not match the requested dataset 'spec_bench'", + ): build_verified_preset( run=run, preset_configuration=PresetConfiguration( @@ -183,7 +269,7 @@ def test_rejects_benchmark_on_a_different_dataset(self, tmp_path): base="Qwen/Qwen3.5-27B", dataset="spec_bench", ), - report=get_successful_preset_report(run), + report=_dataset_report(run, dataset=reported), workspace_path=tmp_path, session_path=tmp_path, preset_id="ab12cd34", @@ -262,3 +348,45 @@ def test_allows_bearer_prose_without_credential(self, tmp_path): assert report.benchmark is not None assert "bearer header" in report.benchmark.command + + +def _shared_prefix_configuration() -> PresetConfiguration: + """The configuration from dstackai/dstack#4198: a shared prefix and no dataset.""" + return PresetConfiguration.model_validate( + { + "type": "preset", + "base": "Qwen/Qwen3.5-27B", + "min_context_length": 262144, + "max_ttft": 5000, + "trials": 4, + "concurrency": 4, + "input_tokens": 131072, + "output_tokens": 512, + "shared_prefix_tokens": 130048, + } + ) + + +def _shared_prefix_report(run: Run, **workload: Any) -> PresetAgentSuccess: + return _report_with_workload(run, {**SHARED_PREFIX_WORKLOAD, **workload}) + + +def _dataset_report(run: Run, **workload: Any) -> PresetAgentSuccess: + # A dataset defines the request shape, so the workload records what it measured. + return _report_with_workload( + run, + { + "api": "chat_completions", + "num_requests": 16, + "input_tokens": 347, + "output_tokens": 2451, + "concurrency": 4, + **workload, + }, + ) + + +def _report_with_workload(run: Run, workload: dict[str, Any]) -> PresetAgentSuccess: + benchmark = get_preset_benchmark() + benchmark.workload = PresetWorkload.model_validate(workload) + return get_successful_preset_report(run).model_copy(update={"benchmark": benchmark})