[Presets] Make a preset workload state the requested dataset

frontend/src/pages/Presets/Details/Constraints/index.tsx

@@ -7,8 +7,6 @@ import { Box, ColumnLayout, Container, Header, Loader } from 'components';
 import { formatTokenCount } from 'libs/presets';
 import { useGetPresetQuery } from 'services/preset';
 
-const DEFAULT_DATASET = 'random';
-
 export const PresetConstraints: FC = () => {
     const { t } = useTranslation();
     const params = useParams();
@@ -37,10 +35,12 @@ export const PresetConstraints: FC = () => {
     return (
         <Container header={<Header variant="h2">{t('presets.constraints')}</Header>}>
             <ColumnLayout columns={4} variant="text-grid">
-                <div>
-                    <Box variant="awsui-key-label">{t('presets.dataset')}</Box>
-                    <div>{dataset ?? DEFAULT_DATASET}</div>
-                </div>
+                {dataset && (
+                    <div>
+                        <Box variant="awsui-key-label">{t('presets.dataset')}</Box>
+                        <div>{dataset}</div>
+                    </div>
+                )}
                 <div>
                     <Box variant="awsui-key-label">{t('presets.input_tokens')}</Box>
                     <div>{formatTokenCount(inputTokens)}</div>

src/dstack/_internal/cli/services/presets/create.py

@@ -74,7 +74,6 @@
 from dstack._internal.cli.utils.offers import print_offers_table
 from dstack._internal.core.errors import CLIError, ConfigurationError
 from dstack._internal.core.models.configurations import (
-    DEFAULT_DATASET,
     PresetConfiguration,
     TaskConfiguration,
 )
@@ -176,9 +175,12 @@ def load_session_configuration(session: PresetSession) -> PresetConfiguration:
             f" followed; run `dstack preset resume {session.preset_id}` instead"
         )
     try:
-        return PresetConfiguration.model_validate(
-            yaml.safe_load(configuration_path.read_text(encoding="utf-8"))
-        )
+        data = yaml.safe_load(configuration_path.read_text(encoding="utf-8"))
+        # A session started before the `random` alias was retired legally saved
+        # it; the rejection is for fresh configurations, not this record.
+        if isinstance(data, dict) and data.get("dataset") == "random":
+            data = {key: value for key, value in data.items() if key != "dataset"}
+        return PresetConfiguration.model_validate(data)
     except (OSError, ValueError) as e:
         raise CLIError(f"Could not read the preset configuration: {e}") from e
 
@@ -615,7 +617,7 @@ async def _create_preset(
         user_prompt=setup.user_prompt,
         baseline=configuration.effective_baseline,
         previous=setup.previous,
-        custom_dataset=configuration.effective_dataset != DEFAULT_DATASET,
+        custom_dataset=configuration.dataset is not None,
     )
     if setup.write_constraints:
         if setup.user_prompt:
@@ -916,8 +918,7 @@ def _build_constraints(
     build_name: str,
     allowed_fleets: Sequence[str],
 ) -> str:
-    dataset = configuration.effective_dataset
-    if dataset == DEFAULT_DATASET:
+    if configuration.dataset is None:
         constraints: PresetConstraints = PresetRandomConstraints(
             run_name_prefix=build_name,
             model=configuration.model,
@@ -940,7 +941,7 @@ def _build_constraints(
             max_ttft=configuration.max_ttft,
             trials_num=configuration.trials,
             concurrency=configuration.concurrency,
-            dataset=dataset,
+            dataset=configuration.dataset,
             baseline=configuration.effective_baseline,
             fleets=list(allowed_fleets),
             env=list(configuration.env),

src/dstack/_internal/cli/services/presets/output.py

@@ -6,7 +6,7 @@
 
 from dstack._internal.cli.models.presets import AnyStoredPreset, VerifiedPreset
 from dstack._internal.cli.utils.common import add_row_from_dict, console
-from dstack._internal.core.models.configurations import DEFAULT_DATASET, PresetConfiguration
+from dstack._internal.core.models.configurations import PresetConfiguration
 from dstack._internal.core.models.presets import PresetWorkload
 from dstack._internal.utils.common import pretty_date, pretty_resources
 
@@ -294,15 +294,14 @@ def format_preset_objective(
     # A pulled preset carries no creation context; the measured workload is its
     # honest record of the conditions the numbers hold for.
     parts = []
-    if workload.dataset != DEFAULT_DATASET:
+    if workload.dataset is not None:
         parts.append(f"data={workload.dataset}")
     else:
         parts.append(
             f"io={_format_token_count(workload.input_tokens)}"
             f"/{_format_token_count(workload.output_tokens)}"
         )
-        shared_prefix_tokens = getattr(workload, "shared_prefix_tokens", 0)
-        share = round(100 * shared_prefix_tokens / workload.input_tokens)
+        share = round(100 * workload.shared_prefix_tokens / workload.input_tokens)
         if share:
             parts.append(f"prefix={share}%")
     parts.append(f"c={workload.concurrency}")
@@ -313,8 +312,8 @@ def _format_creation_objective(
     configuration: PresetConfiguration, workload: PresetWorkload, *, verbose: bool
 ) -> str:
     parts = []
-    if configuration.effective_dataset != DEFAULT_DATASET:
-        parts.append(f"data={configuration.effective_dataset}")
+    if configuration.dataset is not None:
+        parts.append(f"data={configuration.dataset}")
     else:
         input_tokens = configuration.effective_input_tokens
         parts.append(

src/dstack/_internal/cli/services/presets/store.py

@@ -155,6 +155,7 @@ def _load(self, path: Path) -> AnyStoredPreset:
             upgraded = _upgrade_model_field(upgraded)
             upgraded = _upgrade_submitted_at(upgraded)
             upgraded = _upgrade_untagged_preset(upgraded)
+            upgraded = _upgrade_random_dataset_alias(upgraded)
             preset = _STORED_PRESET_ADAPTER.validate_python(upgraded)
         except (OSError, ValidationError, yaml.YAMLError) as e:
             if isinstance(data, dict) and "validations" in data:
@@ -202,6 +203,20 @@ def _upgrade_untagged_preset(data: Any) -> Any:
     return {**data, "status": status}
 
 
+# TODO: Remove in 0.22
+def _upgrade_random_dataset_alias(data: Any) -> Any:
+    """`random` used to be the explicit way to request synthetic prompts, and its
+    own validator suggested writing it; the retired alias is rejected in a fresh
+    configuration but cannot invalidate a record it was legal in."""
+    configuration = data.get("configuration") if isinstance(data, dict) else None
+    if not isinstance(configuration, dict) or configuration.get("dataset") != "random":
+        return data
+    return {
+        **data,
+        "configuration": {key: value for key, value in configuration.items() if key != "dataset"},
+    }
+
+
 # TODO: Remove in 0.22
 def _upgrade_pre_0_21_2_preset(data: dict, *, preset_id: str) -> dict:
     """A preset file from before 0.21.2 stored the service as `service`, the

src/dstack/_internal/cli/services/presets/verify.py

@@ -27,7 +27,11 @@
 from dstack._internal.core.errors import CLIError
 from dstack._internal.core.models.configurations import PresetConfiguration, ServiceConfiguration
 from dstack._internal.core.models.envs import EnvSentinel
-from dstack._internal.core.models.presets import PresetVerificationReplicaGroup
+from dstack._internal.core.models.presets import (
+    PresetBenchmark,
+    PresetVerificationReplicaGroup,
+    PresetWorkload,
+)
 from dstack._internal.core.models.runs import JobStatus, Run, RunStatus
 
 
@@ -110,7 +114,7 @@ def build_verified_preset(
         # it cannot be confused with the service's client-facing model name.
         repo=report.model,
         context_length=report.context_length,
-        benchmark=report.benchmark,
+        benchmark=_normalized_benchmark(report.benchmark, preset_configuration),
         best_trial=report.trial,
         configuration=preset_configuration,
         preset_id=preset_id,
@@ -134,18 +138,63 @@ def _verified_run_service(run: Run, report: PresetAgentSuccess) -> ServiceConfig
 def _check_report_answers_request(
     report: PresetAgentSuccess, configuration: PresetConfiguration
 ) -> None:
-    """The report must answer what the configuration asked: the same dataset, and
-    the requested model — exactly when it was exact, any variant of the base
-    otherwise."""
-    if report.benchmark.workload.dataset != configuration.effective_dataset:
-        raise CLIError("Claude final benchmark dataset does not match the requested dataset")
+    """The report must answer what the configuration asked: the same benchmark
+    workload, and the requested model — exactly when it was exact, any variant of
+    the base otherwise."""
+    _check_workload_answers_request(report.benchmark.workload, configuration)
     if configuration.model.allows_variant_selection:
         if report.base != configuration.model.api_model_name:
             raise CLIError("Claude final report base does not match the requested model")
     elif report.model != configuration.model.exact_repo:
         raise CLIError("Claude changed an exact model request")
 
 
+def _check_workload_answers_request(
+    workload: PresetWorkload, configuration: PresetConfiguration
+) -> None:
+    """Only what the configuration specifies exactly is compared. A named dataset is
+    compared by name, because the request and the report both use the dataset's own
+    name. A synthetic workload has no such shared name — the report's `dataset`, if
+    any, is the benchmark tool's own name for the data it generated — so the shared
+    prefix it was run with is compared instead. `input_tokens` and `output_tokens`
+    are what the benchmark measured rather than an echo of the request, so they are
+    not compared."""
+    if configuration.dataset is not None:
+        if workload.dataset != configuration.dataset:
+            raise CLIError(
+                f"Claude final benchmark dataset {workload.dataset!r} does not match the"
+                f" requested dataset {configuration.dataset!r}"
+            )
+    else:
+        shared_prefix_tokens = configuration.shared_prefix_tokens or 0
+        if workload.shared_prefix_tokens != shared_prefix_tokens:
+            raise CLIError(
+                f"Claude final benchmark shared prefix of {workload.shared_prefix_tokens}"
+                f" tokens does not match the requested {shared_prefix_tokens}"
+            )
+    if configuration.concurrency is not None and workload.concurrency != configuration.concurrency:
+        raise CLIError(
+            f"Claude final benchmark concurrency of {workload.concurrency} does not match the"
+            f" requested concurrency of {configuration.concurrency}"
+        )
+
+
+def _normalized_benchmark(
+    benchmark: PresetBenchmark, configuration: PresetConfiguration
+) -> PresetBenchmark:
+    """The stored workload states the request, and the configuration is the
+    authority on what was requested: a synthetic run carries no dataset — whatever
+    the benchmark tool called its generated data is already on record in `command`
+    — and a dataset run carries no shared prefix. The agent's report may volunteer
+    either; neither is trusted into the record."""
+    workload = benchmark.workload.model_copy(
+        update=(
+            {"dataset": None} if configuration.dataset is None else {"shared_prefix_tokens": 0}
+        )
+    )
+    return benchmark.model_copy(update={"workload": workload})
+
+
 def _portable_service(
     service: ServiceConfiguration,
     configuration: PresetConfiguration,

src/dstack/_internal/core/models/configurations.py

@@ -1593,7 +1593,6 @@ def replica_groups(self) -> List[ReplicaGroup]:
 DEFAULT_INPUT_TOKENS = 1024
 DEFAULT_OUTPUT_TOKENS = 1024
 DEFAULT_BASELINE = True
-DEFAULT_DATASET = "random"
 
 
 class PresetModelRepo(CoreModel):
@@ -1794,10 +1793,9 @@ class PresetConfiguration(
         Optional[str],
         Field(
             description=(
-                "The benchmark dataset used during preset creation: `random` for synthetic"
-                " prompts shaped by `input_tokens` and `output_tokens`, a benchmark tool's"
-                " dataset name (e.g. `sharegpt`, `spec_bench`), or a Hugging Face dataset ID."
-                " Defaults to `random`"
+                "The benchmark dataset used during preset creation: a benchmark tool's"
+                " dataset name (e.g. `sharegpt`, `spec_bench`) or a Hugging Face dataset ID."
+                " Omit for synthetic prompts shaped by `input_tokens` and `output_tokens`"
             )
         ),
     ] = None
@@ -1838,10 +1836,6 @@ def effective_output_tokens(self) -> int:
     def effective_baseline(self) -> bool:
         return self.baseline if self.baseline is not None else DEFAULT_BASELINE
 
-    @property
-    def effective_dataset(self) -> str:
-        return self.dataset if self.dataset is not None else DEFAULT_DATASET
-
     @field_validator("dataset")
     @classmethod
     def validate_dataset_name(cls, value: Optional[str]) -> Optional[str]:
@@ -1852,11 +1846,15 @@ def validate_dataset_name(cls, value: Optional[str]) -> Optional[str]:
         value = value.strip()
         if not value:
             raise ValueError("dataset must be a non-empty string")
+        if value == "random":
+            # The retired alias for the default. A set dataset now always means a
+            # real one; synthetic prompts are requested by omitting it.
+            raise ValueError("`random` is not a dataset; omit `dataset` for synthetic prompts")
         return value
 
     @model_validator(mode="after")
     def validate_dataset(self) -> Self:
-        if self.dataset in (None, DEFAULT_DATASET):
+        if self.dataset is None:
             return self
         set_fields = [
             name
@@ -1865,8 +1863,8 @@ def validate_dataset(self) -> Self:
         ]
         if set_fields:
             raise ValueError(
-                f"{', '.join(set_fields)} can only be set with the `random` dataset;"
-                " a custom dataset defines its own request shape"
+                f"{', '.join(set_fields)} shape synthetic prompts and cannot be set"
+                " together with `dataset`; a dataset defines its own request shape"
             )
         return self
 

src/dstack/_internal/core/models/presets.py

@@ -1,5 +1,5 @@
 import re
-from typing import Any, List, Literal, Optional, Sequence, Union
+from typing import Any, List, Literal, Optional, Sequence
 
 from pydantic import Field, PositiveFloat, PositiveInt, field_validator, model_validator
 from typing_extensions import Annotated, Self
@@ -36,18 +36,34 @@ class PresetWorkload(CoreModel):
     # A Literal, not a str: the agent-facing JSON schema is generated from this
     # model, so the allowed values must be part of it.
     api: Literal["chat_completions", "completions"]
-    dataset: str
+    dataset: Optional[str] = None
+    """The dataset the configuration requested, which the benchmark served.
+    Absent for a synthetic workload: no dataset was requested, and the name the
+    benchmark tool gives the data it generates is its own affair, on record in
+    `command`."""
     num_requests: PositiveInt
     input_tokens: PositiveInt
     output_tokens: Annotated[int, Field(ge=2)]
     concurrency: PositiveInt
-
-
-class PresetRandomWorkload(PresetWorkload):
-    # A Literal, not a defaulted str: `PresetBenchmark.workload` is a union with
-    # the base model, and only a literal `dataset` discriminates it.
-    dataset: Literal["random"] = "random"
     shared_prefix_tokens: Annotated[int, Field(ge=0)] = 0
+    """How many leading tokens every measured request shared. Only a synthetic
+    workload has one: a named dataset defines its own requests."""
+
+    @model_validator(mode="before")
+    @classmethod
+    def drop_legacy_random_dataset(cls, data: Any) -> Any:
+        # Before the dataset meant the requested one, every synthetic workload
+        # was stored as the literal `random` next to its shared prefix. The
+        # combination can mean nothing else: `random` is rejected as a
+        # configuration dataset, and a dataset defines its own requests, so a
+        # dataset workload never records a prefix.
+        if (
+            isinstance(data, dict)
+            and data.get("dataset") == "random"
+            and "shared_prefix_tokens" in data
+        ):
+            data = {key: value for key, value in data.items() if key != "dataset"}
+        return data
 
 
 class PresetBenchmarkLatency(CoreModel):
@@ -78,9 +94,7 @@ class PresetBenchmark(CoreModel):
     tool: Annotated[str, Field(min_length=1)]
     tool_version: Annotated[str, Field(min_length=1)]
     command: Annotated[str, Field(min_length=1)]
-    # The subclass first: a report without `dataset` is a random workload, and a
-    # base-typed field would reject its `shared_prefix_tokens` as unknown.
-    workload: Union[PresetRandomWorkload, PresetWorkload]
+    workload: PresetWorkload
     metrics: PresetBenchmarkMetrics
 
     @property

src/tests/_internal/cli/common.py

@@ -79,6 +79,20 @@ def run_dstack_cli(
     return exit_code
 
 
+# The trial workload from the SGLang session in dstackai/dstack#4198: a synthetic
+# shared-prefix benchmark run with a tool that does not call its generated data
+# `random`. It used to match neither of the two workload models the schema offered.
+SHARED_PREFIX_WORKLOAD = {
+    "api": "completions",
+    "dataset": "generated-shared-prefix",
+    "num_requests": 16,
+    "input_tokens": 131072,
+    "output_tokens": 512,
+    "concurrency": 4,
+    "shared_prefix_tokens": 130048,
+}
+
+
 def get_preset_benchmark() -> PresetBenchmark:
     benchmark = PresetBenchmark(
         tool="vllm bench serve",