diff --git a/.github/actions/setup-bun/action.yml b/.github/actions/setup-bun/action.yml index 9d29724d86a4..92bddf7f531a 100644 --- a/.github/actions/setup-bun/action.yml +++ b/.github/actions/setup-bun/action.yml @@ -15,12 +15,15 @@ runs: with: node-version: "24" - - name: Get baseline download URL + - name: Get Bun version and baseline download URL id: bun-url shell: bash run: | + V=$(node -p "require('./package.json').packageManager.split('@')[1]") + # Bun 1.4.2 includes the patched peer-variant fix in oven-sh/bun#33646. + if [ "$RUNNER_OS" = "Windows" ]; then V=1.4.2; fi + echo "version=$V" >> "$GITHUB_OUTPUT" if [ "$RUNNER_ARCH" = "X64" ]; then - V=$(node -p "require('./package.json').packageManager.split('@')[1]") case "$RUNNER_OS" in macOS) OS=darwin ;; Linux) OS=linux ;; @@ -30,9 +33,10 @@ runs: fi - name: Setup Bun + id: setup-bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: - bun-version-file: ${{ !steps.bun-url.outputs.url && 'package.json' || '' }} + bun-version: ${{ !steps.bun-url.outputs.url && steps.bun-url.outputs.version || '' }} bun-download-url: ${{ steps.bun-url.outputs.url }} - name: Get cache directory @@ -45,24 +49,16 @@ runs: uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ steps.cache.outputs.dir }} - key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + key: ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-version }}-${{ hashFiles('**/bun.lock') }} restore-keys: | - ${{ runner.os }}-bun- + ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-version }}- - name: Install setuptools for distutils compatibility run: python3 -m pip install setuptools || pip install setuptools || true shell: bash - name: Install dependencies - run: | - # Workaround for patched peer variants - # e.g. ./patches/ for standard-openapi - # https://github.com/oven-sh/bun/issues/28147 - if [ "$RUNNER_OS" = "Windows" ]; then - bun install --linker hoisted ${{ inputs.install-flags }} - else - bun install ${{ inputs.install-flags }} - fi + run: bun install ${{ inputs.install-flags }} shell: bash - name: Save Bun dependencies @@ -70,4 +66,4 @@ runs: uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ steps.cache.outputs.dir }} - key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + key: ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-version }}-${{ hashFiles('**/bun.lock') }} diff --git a/packages/stats/app/src/routes/compare-radar.tsx b/packages/stats/app/src/routes/compare-radar.tsx index 35ada12885f1..3ed23586b949 100644 --- a/packages/stats/app/src/routes/compare-radar.tsx +++ b/packages/stats/app/src/routes/compare-radar.tsx @@ -22,6 +22,12 @@ type RadarAxis = { label: string description: string score: (model: ModelCatalogEntry) => number | undefined + capability?: "reasoning" | "toolCall" +} + +type RadarScore = { + value: number + fallback?: string } type RadarPoint = { @@ -37,7 +43,7 @@ export function ComparisonRadar(props: ComparisonRadarProps) { name: model.name, labName: model.labName, color: radarColors[index % radarColors.length], - scores: axes().map((axis) => (model.catalog ? axis.score(model.catalog) : undefined)), + scores: axes().map((axis) => resolveRadarScore(axis, model.catalog)), })), ) const accessibleDescription = createMemo(() => @@ -62,6 +68,9 @@ export function ComparisonRadar(props: ComparisonRadarProps) { {model.name} {(name) => {name()}} + score.fallback)}> + Hollow points use fallbacks · hover for details + )} @@ -89,10 +98,16 @@ export function ComparisonRadar(props: ComparisonRadarProps) { {(score, index) => { - const point = () => radarPoint(index(), axes().length, score ?? 0) + const point = () => radarPoint(index(), axes().length, score.value) return ( <> - + {axes()[activeAxis() ?? 0]?.label}

{axes()[activeAxis() ?? 0]?.description}

+ + {(model) => ( +

+ {model.name}: {formatRadarScore(model.scores[activeAxis() ?? 0])} +

+ )} + @@ -166,7 +188,7 @@ export function ComparisonRadar(props: ComparisonRadarProps) { ) } -function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[] { +export function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[] { const benchmarks = benchmarkScoreGroups(catalogModels) const toolUseBenchmarks = benchmarkScoreGroups(catalogModels, true) const costs = catalogModels.flatMap((model) => { @@ -180,9 +202,10 @@ function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[ return [ { label: "Reasoning", - description: "Ability to solve complex, multi-step problems. Based on reasoning benchmarks when available.", - score: (model) => - benchmarkPercentile(model, benchmarks, reasoningBenchmarkPattern) ?? (model.reasoning ? 100 : 0), + capability: "reasoning", + description: + "Ability to solve complex, multi-step problems. Benchmarks take priority; reasoning support defaults to 50/100.", + score: (model) => benchmarkPercentile(model, benchmarks, reasoningBenchmarkPattern), }, { label: "Coding", @@ -218,7 +241,8 @@ function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[ }, { label: "Tool use", - description: "Performance on agent benchmarks including Terminal-Bench, Tau3, and Claw-Eval.", + capability: "toolCall", + description: "Agent benchmark performance. Benchmarks take priority; tool calling support defaults to 50/100.", score: (model) => benchmarkPercentile(model, toolUseBenchmarks, toolUseBenchmarkPattern, { aggregate: "average", @@ -324,9 +348,9 @@ function radarPolygonPoints(count: number, score: number) { .join(" ") } -function radarSeriesPolygon(scores: (number | undefined)[]) { +function radarSeriesPolygon(scores: RadarScore[]) { return scores - .map((score, index) => radarPoint(index, scores.length, score ?? 0)) + .map((score, index) => radarPoint(index, scores.length, score.value)) .map((point) => `${point.x},${point.y}`) .join(" ") } @@ -355,6 +379,17 @@ function roundRadarCoordinate(value: number) { return Math.round(value * 1000) / 1000 } -function formatRadarScore(score: number | undefined) { - return score === undefined ? "No data" : `${Math.round(score)}/100` +function formatRadarScore(score: RadarScore) { + return `${Math.round(score.value)}/100${score.fallback ? ` — ${score.fallback}` : ""}` +} + +export function resolveRadarScore(axis: RadarAxis, model: ModelCatalogEntry | null): RadarScore { + const score = model ? axis.score(model) : undefined + if (score !== undefined) return { value: score } + const supported = axis.capability ? model?.[axis.capability] : undefined + if (supported === undefined) return { value: 50, fallback: "No data; neutral placeholder" } + const capability = axis.capability === "toolCall" ? "Tool calling" : "Reasoning" + return supported + ? { value: 50, fallback: `${capability} supported; no comparable benchmark` } + : { value: 0, fallback: `${capability} not supported` } } diff --git a/packages/stats/app/src/routes/index.css b/packages/stats/app/src/routes/index.css index e3e2de48e08a..9cc3bfe13b82 100644 --- a/packages/stats/app/src/routes/index.css +++ b/packages/stats/app/src/routes/index.css @@ -6291,8 +6291,7 @@ body { vector-effect: non-scaling-stroke; } -[data-page="stats"] [data-slot="compare-radar-area"], -[data-page="stats"] [data-slot="compare-radar-line"] { +[data-page="stats"] [data-slot="compare-radar-area"] { stroke: currentColor; stroke-width: 1.5px; stroke-linejoin: round; @@ -6304,10 +6303,6 @@ body { fill-opacity: 0.09; } -[data-page="stats"] [data-slot="compare-radar-line"] { - fill: none; -} - [data-page="stats"] [data-slot="compare-radar-point"] { fill: currentColor; stroke: currentColor; @@ -6315,6 +6310,10 @@ body { vector-effect: non-scaling-stroke; } +[data-page="stats"] [data-slot="compare-radar-point"][data-fallback="true"] { + fill: var(--stats-bg); +} + [data-page="stats"] [data-slot="compare-radar-point-hit"] { fill: transparent; cursor: pointer; diff --git a/packages/stats/app/src/routes/model-catalog.ts b/packages/stats/app/src/routes/model-catalog.ts index c55af8594ce8..ae55eb55a851 100644 --- a/packages/stats/app/src/routes/model-catalog.ts +++ b/packages/stats/app/src/routes/model-catalog.ts @@ -25,8 +25,8 @@ export type ModelCatalogEntry = { limit?: { context?: number; output?: number } modalities: { input: string[]; output: string[] } openWeights: boolean - reasoning: boolean - toolCall: boolean + reasoning?: boolean + toolCall?: boolean attachment: boolean temperature: boolean cost?: ModelCatalogCost @@ -54,6 +54,7 @@ export type ModelCatalogLab = { export type ModelCatalog = { models: ModelCatalogEntry[] + aliases?: ModelCatalogEntry[] labs: ModelCatalogLab[] } @@ -80,7 +81,8 @@ export function findModelCatalogEntry(catalog: ModelCatalog, model: string, lab? return ( catalog.models.find((entry) => entry.id.toLowerCase() === normalizedId) ?? catalog.models.find((entry) => (lab ? entry.lab === catalogLabSlug(lab) : true) && entry.slug === leaf) ?? - catalog.models.find((entry) => entry.slug === leaf) + catalog.models.find((entry) => entry.slug === leaf) ?? + catalog.aliases?.find((entry) => (lab ? entry.lab === catalogLabSlug(lab) : true) && entry.slug === leaf) ) } @@ -133,7 +135,7 @@ export function catalogSlug(value: string) { .replace(/-{2,}/g, "-") } -function buildModelCatalog(payload: unknown, pricingPayload?: unknown, labPayload?: unknown): ModelCatalog { +export function buildModelCatalog(payload: unknown, pricingPayload?: unknown, labPayload?: unknown): ModelCatalog { const costs = readCatalogCosts(pricingPayload) const labDescriptions = readCatalogLabDescriptions(payload, pricingPayload, labPayload) const models = readCatalogModels(payload) @@ -149,6 +151,25 @@ function buildModelCatalog(payload: unknown, pricingPayload?: unknown, labPayloa .toSorted((a, b) => a.lab.localeCompare(b.lab) || displayDateTime(b.releaseDate) - displayDateTime(a.releaseDate)) return { models, + // Contributor is a serving tier of these Muse models, with its own pricing. + // Keep aliases out of the model population used to normalize benchmark scores. + aliases: ["meta/muse-spark-1.2", "meta/muse-spark-1.3"].flatMap((id) => { + const model = models.find((entry) => entry.id === id) + if (!model) return [] + const alias = `${id}-contributor` + return [ + { + ...model, + id: alias, + slug: `${model.slug}-contributor`, + name: `${model.name} Contributor`, + cost: + costs.get(catalogIdKey(alias)) ?? + costs.get(`${model.lab}/${model.slug}-contributor`) ?? + costs.get(`${model.slug}-contributor`), + }, + ] + }), labs: Object.values( models.reduce>((result, model) => { result[model.lab] = { @@ -184,8 +205,8 @@ function readModelCatalogEntry(value: unknown): ModelCatalogEntry[] { limit: readCatalogLimit(value.limit), modalities: readCatalogModalities(value.modalities), openWeights: booleanValue(value.open_weights), - reasoning: booleanValue(value.reasoning), - toolCall: booleanValue(value.tool_call), + reasoning: typeof value.reasoning === "boolean" ? value.reasoning : undefined, + toolCall: typeof value.tool_call === "boolean" ? value.tool_call : undefined, attachment: booleanValue(value.attachment), temperature: booleanValue(value.temperature), cost: readCatalogCost(value.cost), diff --git a/packages/stats/app/test/compare-radar.test.ts b/packages/stats/app/test/compare-radar.test.ts new file mode 100644 index 000000000000..59f855c69214 --- /dev/null +++ b/packages/stats/app/test/compare-radar.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test" +import { buildRadarAxes, resolveRadarScore } from "../src/routes/compare-radar" +import type { ModelCatalogEntry } from "../src/routes/model-catalog" + +const model: ModelCatalogEntry = { + id: "meta/muse-spark-1.3", + lab: "meta", + slug: "muse-spark-1-3", + name: "Muse Spark 1.3", + modalities: { input: ["text", "image"], output: ["text"] }, + reasoning: true, + toolCall: true, + openWeights: false, + attachment: true, + temperature: true, + weights: [], + benchmarks: [], +} + +function scores(entry: ModelCatalogEntry | null, catalog = [model]) { + return Object.fromEntries(buildRadarAxes(catalog).map((axis) => [axis.label, resolveRadarScore(axis, entry)])) +} + +describe("radar capability fallbacks", () => { + test("supported capabilities have visible baselines without benchmarks", () => { + const result = scores(model) + expect(result["Tool use"]).toEqual({ value: 50, fallback: "Tool calling supported; no comparable benchmark" }) + expect(result.Reasoning).toEqual({ value: 50, fallback: "Reasoning supported; no comparable benchmark" }) + expect(result.Coding).toEqual({ value: 50, fallback: "No data; neutral placeholder" }) + }) + + test("explicitly unsupported capabilities remain zero", () => { + const result = scores({ ...model, reasoning: false, toolCall: false }) + expect(result["Tool use"]).toEqual({ value: 0, fallback: "Tool calling not supported" }) + expect(result.Reasoning).toEqual({ value: 0, fallback: "Reasoning not supported" }) + }) + + test("unknown capabilities and unmatched models use neutral placeholders", () => { + const result = scores({ ...model, reasoning: undefined, toolCall: undefined }) + expect(result["Tool use"]).toEqual({ value: 50, fallback: "No data; neutral placeholder" }) + expect(result.Reasoning).toEqual({ value: 50, fallback: "No data; neutral placeholder" }) + expect(Object.values(scores(null))).toEqual(Array(6).fill({ value: 50, fallback: "No data; neutral placeholder" })) + }) + + test("measured benchmark percentiles override fallbacks, including zero", () => { + const low = { + ...model, + benchmarks: [ + { name: "Tau3", score: 20 }, + { name: "GPQA", score: 40 }, + ], + } + const high = { + ...model, + id: "other/model", + benchmarks: [ + { name: "Tau3", score: 80 }, + { name: "GPQA", score: 90 }, + ], + } + expect(scores(low, [low, high])["Tool use"]).toEqual({ value: 0 }) + expect(scores(low, [low, high]).Reasoning).toEqual({ value: 0 }) + expect(scores(high, [low, high])["Tool use"]).toEqual({ value: 100 }) + expect(scores(high, [low, high]).Reasoning).toEqual({ value: 100 }) + }) + + test("a benchmark without comparison peers retains the capability baseline", () => { + const entry = { ...model, benchmarks: [{ name: "Tau3", score: 90 }] } + expect(scores(entry, [entry])["Tool use"]).toEqual({ + value: 50, + fallback: "Tool calling supported; no comparable benchmark", + }) + }) +})