From 463e76557daa6dccfa2b5750db60dad5121eb4c3 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 12 Aug 2026 11:51:10 -0700 Subject: [PATCH 01/10] feat(benchmarks): add cross-process TPM rate limiter - Add createRateLimiter with a shared SQLite rolling-window ledger - Reserve then settle token claims so concurrent processes stay under quota - Fail closed when a limited model has no positive token estimate - Export the limiter from @typeagent/benchmarks and cover admit/throttle paths --- .../benchmarks/src/core/rateLimiter.ts | 318 ++++++++++++++++++ ts/packages/benchmarks/src/index.ts | 1 + .../test/translationBench.rateLimiter.spec.ts | 145 ++++++++ 3 files changed, 464 insertions(+) create mode 100644 ts/packages/benchmarks/src/core/rateLimiter.ts create mode 100644 ts/packages/benchmarks/test/translationBench.rateLimiter.spec.ts diff --git a/ts/packages/benchmarks/src/core/rateLimiter.ts b/ts/packages/benchmarks/src/core/rateLimiter.ts new file mode 100644 index 0000000000..344c291823 --- /dev/null +++ b/ts/packages/benchmarks/src/core/rateLimiter.ts @@ -0,0 +1,318 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import fs from "node:fs"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { DatabaseSync, type StatementSync } from "node:sqlite"; + +const WINDOW_MS = 60_000; +const MAX_SLEEP_MS = 1_000; +// Long enough for multi-minute TB translates + retries; pending claims older +// than this are treated as abandoned (process crash) and purged. +const STALE_MS = 30 * 60_000; +const BUSY_TIMEOUT_MS = 15_000; +const SQLITE_BUSY = 5; +const OPEN_MAX_ATTEMPTS = 50; +const OPEN_RETRY_MIN_MS = 20; +const OPEN_RETRY_JITTER_MS = 30; + +export interface RateLimiterOptions { + dbPath: string; + estTokensPerCall?: number; + maxWaitMs?: number; + onWait?: (model: string, waitedMs: number, waitMs: number) => void; +} + +export interface RateLimiter { + disabledFor(model: string): boolean; + run( + model: string, + est: number | undefined, + fn: () => Promise<{ result: T; actualTokens: number | undefined }>, + ): Promise; + close(): void; +} + +export type TpmLimits = Readonly>; + +interface Reservation { + id: string | undefined; + waitMs: number; +} + +interface ClaimRow { + created_at: number; + tokens: number; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isBusyError(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + (error as { errcode?: number }).errcode === SQLITE_BUSY + ); +} + +function openDatabase(dbPath: string): DatabaseSync { + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + let lastError: unknown; + for (let attempt = 0; attempt < OPEN_MAX_ATTEMPTS; attempt++) { + let db: DatabaseSync | undefined; + try { + db = new DatabaseSync(dbPath); + db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`); + db.exec("PRAGMA journal_mode = WAL"); + db.exec("PRAGMA synchronous = NORMAL"); + db.exec( + "CREATE TABLE IF NOT EXISTS claims (" + + "id TEXT PRIMARY KEY, " + + "model TEXT NOT NULL, " + + "tokens REAL NOT NULL, " + + "created_at INTEGER NOT NULL, " + + "pending INTEGER NOT NULL)", + ); + db.exec( + "CREATE INDEX IF NOT EXISTS idx_claims_model_time " + + "ON claims (model, created_at)", + ); + return db; + } catch (error) { + lastError = error; + if (db !== undefined) { + try { + db.close(); + } catch { + // no-op + } + } + if (!isBusyError(error)) { + throw error; + } + const until = + Date.now() + + OPEN_RETRY_MIN_MS + + Math.floor(Math.random() * OPEN_RETRY_JITTER_MS); + // Yield the event loop instead of a tight spin-wait. + const sab = new SharedArrayBuffer(4); + Atomics.wait( + new Int32Array(sab), + 0, + 0, + Math.max(1, until - Date.now()), + ); + } + } + throw lastError; +} + +class Ledger { + private readonly insertStmt: StatementSync; + private readonly settleStmt: StatementSync; + private readonly insertSettledStmt: StatementSync; + private readonly purgeExpiredStmt: StatementSync; + private readonly purgeStaleStmt: StatementSync; + private readonly usedStmt: StatementSync; + private readonly oldestStmt: StatementSync; + + constructor( + private readonly db: DatabaseSync, + private readonly tpmLimits: TpmLimits, + ) { + this.insertStmt = db.prepare( + "INSERT INTO claims (id, model, tokens, created_at, pending) " + + "VALUES (?, ?, ?, ?, 1)", + ); + this.settleStmt = db.prepare( + "UPDATE claims SET tokens = ?, pending = 0 WHERE id = ?", + ); + this.insertSettledStmt = db.prepare( + "INSERT OR REPLACE INTO claims " + + "(id, model, tokens, created_at, pending) VALUES (?, ?, ?, ?, 0)", + ); + this.purgeExpiredStmt = db.prepare( + "DELETE FROM claims WHERE pending = 0 AND created_at <= ?", + ); + this.purgeStaleStmt = db.prepare( + "DELETE FROM claims WHERE pending = 1 AND created_at <= ?", + ); + this.usedStmt = db.prepare( + "SELECT COALESCE(SUM(tokens), 0) AS used " + + "FROM claims WHERE model = ? AND created_at > ?", + ); + this.oldestStmt = db.prepare( + "SELECT created_at, tokens FROM claims " + + "WHERE model = ? AND created_at > ? ORDER BY created_at ASC", + ); + } + + private transaction(fn: () => T): T { + this.db.exec("BEGIN IMMEDIATE"); + try { + const out = fn(); + this.db.exec("COMMIT"); + return out; + } catch (error) { + try { + this.db.exec("ROLLBACK"); + } catch { + // no-op + } + throw error; + } + } + + private waitForCapacity( + model: string, + limit: number, + need: number, + now: number, + ): number { + const excess = need - limit; + let freed = 0; + const rows = this.oldestStmt.all( + model, + now - WINDOW_MS, + ) as unknown as ClaimRow[]; + for (const row of rows) { + freed += row.tokens; + if (freed >= excess) { + return Math.max(5, row.created_at + WINDOW_MS - now); + } + } + return Math.max(5, WINDOW_MS); + } + + reserve(model: string, cost: number): Reservation { + const limit = this.tpmLimits[model]; + const need = Math.min(cost, limit); + return this.transaction(() => { + const now = Date.now(); + this.purgeExpiredStmt.run(now - WINDOW_MS); + this.purgeStaleStmt.run(now - STALE_MS); + const { used } = this.usedStmt.get(model, now - WINDOW_MS) as { + used: number; + }; + if (used + need <= limit) { + const id = randomUUID(); + this.insertStmt.run(id, model, need, now); + return { id, waitMs: 0 }; + } + return { + id: undefined, + waitMs: this.waitForCapacity(model, limit, used + need, now), + }; + }); + } + + settle(id: string, model: string, actualCost: number): void { + this.transaction(() => { + const result = this.settleStmt.run(actualCost, id); + if (result.changes === 0) { + this.insertSettledStmt.run(id, model, actualCost, Date.now()); + } + }); + } +} + +export function createRateLimiter( + limits: TpmLimits, + options: RateLimiterOptions, +): RateLimiter { + const tpmLimits: Record = {}; + for (const [model, tpm] of Object.entries(limits)) { + if (Number.isFinite(tpm) && tpm > 0) { + tpmLimits[model] = tpm; + } + } + + let db: DatabaseSync | undefined; + let ledger: Ledger | undefined; + if (Object.keys(tpmLimits).length > 0) { + db = openDatabase(options.dbPath); + ledger = new Ledger(db, tpmLimits); + } + + async function admit(model: string, estCost: number): Promise { + const activeLedger = ledger as Ledger; + const startedAt = Date.now(); + for (;;) { + const reservation = activeLedger.reserve(model, estCost); + if (reservation.id !== undefined) { + return reservation.id; + } + const waited = Date.now() - startedAt; + if ( + options.maxWaitMs !== undefined && + waited >= options.maxWaitMs + ) { + throw new Error( + `rate limiter: exceeded max wait ${options.maxWaitMs}ms for ${model}`, + ); + } + options.onWait?.(model, waited, reservation.waitMs); + await sleep(Math.min(reservation.waitMs, MAX_SLEEP_MS)); + } + } + + async function run( + model: string, + est: number | undefined, + fn: () => Promise<{ result: T; actualTokens: number | undefined }>, + ): Promise { + if (ledger === undefined || tpmLimits[model] === undefined) { + return (await fn()).result; + } + + const estCost = + est !== undefined && Number.isFinite(est) && est > 0 + ? est + : options.estTokensPerCall; + if (estCost === undefined || !(estCost > 0)) { + throw new Error( + `rate limiter: no positive token estimate for ${model}`, + ); + } + + const id = await admit(model, estCost); + let actual = estCost; + try { + const out = await fn(); + actual = + out.actualTokens !== undefined && + Number.isFinite(out.actualTokens) && + out.actualTokens > 0 + ? out.actualTokens + : estCost; + return out.result; + } finally { + try { + (ledger as Ledger).settle(id, model, actual); + } catch (error) { + const message = + error instanceof Error ? error.message : String(error); + console.error( + `[rate-limit] settle failed model=${model} id=${id} actual=${actual}: ${message}`, + ); + } + } + } + + return { + disabledFor(model: string): boolean { + return tpmLimits[model] === undefined; + }, + close(): void { + if (db !== undefined) { + db.close(); + db = undefined; + ledger = undefined; + } + }, + run, + }; +} diff --git a/ts/packages/benchmarks/src/index.ts b/ts/packages/benchmarks/src/index.ts index 7783e80dcd..1e4182b40f 100644 --- a/ts/packages/benchmarks/src/index.ts +++ b/ts/packages/benchmarks/src/index.ts @@ -4,4 +4,5 @@ export * from "./core/paths.js"; export * from "./core/types.js"; export * from "./core/prices.js"; +export * from "./core/rateLimiter.js"; export * from "./translationBench/index.js"; diff --git a/ts/packages/benchmarks/test/translationBench.rateLimiter.spec.ts b/ts/packages/benchmarks/test/translationBench.rateLimiter.spec.ts new file mode 100644 index 0000000000..940e2fba00 --- /dev/null +++ b/ts/packages/benchmarks/test/translationBench.rateLimiter.spec.ts @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, beforeEach, describe, expect, it } from "@jest/globals"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { + createRateLimiter, + type RateLimiter, +} from "../src/core/rateLimiter.js"; + +describe("translationBench rateLimiter", () => { + let tempDir: string; + let dbPath: string; + const limiters: RateLimiter[] = []; + + beforeEach(() => { + tempDir = mkdtempSync(path.join(tmpdir(), "tb-ratelimiter-")); + dbPath = path.join(tempDir, "tpm.sqlite"); + }); + + afterEach(() => { + while (limiters.length > 0) { + limiters.pop()?.close(); + } + rmSync(tempDir, { recursive: true, force: true }); + }); + + function make( + limits: Record, + estTokensPerCall = 10_400, + ): RateLimiter { + const limiter = createRateLimiter(limits, { + dbPath, + estTokensPerCall, + maxWaitMs: 300, + }); + limiters.push(limiter); + return limiter; + } + + it("passes through models without a positive quota", async () => { + const limiter = make({ "azure/free": 0, "azure/missing": NaN }); + expect(limiter.disabledFor("azure/free")).toBe(true); + expect(limiter.disabledFor("azure/unknown")).toBe(true); + + const result = await limiter.run("azure/free", 1000, async () => ({ + result: "ok", + actualTokens: 1000, + })); + expect(result).toBe("ok"); + }); + + it("admits calls that fit within the per-minute budget", async () => { + const limiter = make({ "azure/m": 600_000 }); + expect(limiter.disabledFor("azure/m")).toBe(false); + + let calls = 0; + for (let i = 0; i < 10; i++) { + await limiter.run("azure/m", 1000, async () => { + calls++; + return { result: calls, actualTokens: 1000 }; + }); + } + expect(calls).toBe(10); + }); + + it("throttles a call that would exceed the budget", async () => { + const limiter = make({ "azure/m": 120_000 }); + + await limiter.run("azure/m", 100_000, async () => ({ + result: "big", + actualTokens: 100_000, + })); + + await expect( + limiter.run("azure/m", 30_000, async () => ({ + result: "blocked", + actualTokens: 30_000, + })), + ).rejects.toThrow(/max wait/); + }); + + it("settles claims to the measured actual token count", async () => { + const limiter = make({ "azure/m": 120_000 }); + + await limiter.run("azure/m", 100_000, async () => ({ + result: "over-estimated", + actualTokens: 10_000, + })); + + let admittedPromptly = false; + await limiter.run("azure/m", 100_000, async () => { + admittedPromptly = true; + return { result: "second", actualTokens: 10_000 }; + }); + expect(admittedPromptly).toBe(true); + }); + + it("shares one budget across independent limiter instances (same db)", async () => { + const a = make({ "azure/m": 120_000 }); + const b = make({ "azure/m": 120_000 }); + + await a.run("azure/m", 100_000, async () => ({ + result: "a", + actualTokens: 100_000, + })); + + await expect( + b.run("azure/m", 30_000, async () => ({ + result: "b", + actualTokens: 30_000, + })), + ).rejects.toThrow(/max wait/); + }); + + it("falls back to the default estimate when none is given", async () => { + const limiter = make({ "azure/m": 60_000 }, 50_000); + + let calls = 0; + for (let i = 0; i < 3; i++) { + await limiter.run("azure/m", undefined, async () => { + calls++; + return { result: calls, actualTokens: 1_000 }; + }); + } + expect(calls).toBe(3); + }); + + it("throws when no positive estimate is available for a limited model", async () => { + const limiter = createRateLimiter( + { "azure/m": 120_000 }, + { dbPath, estTokensPerCall: 0 }, + ); + limiters.push(limiter); + await expect( + limiter.run("azure/m", undefined, async () => ({ + result: "x", + actualTokens: 1, + })), + ).rejects.toThrow(/token estimate/); + }); +}); From 61220e27b7489e95f17b905acbdae871d1d4190b Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Wed, 12 Aug 2026 19:04:26 +0000 Subject: [PATCH 02/10] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index ecc64a825c..d67aa95841 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -49,13 +49,13 @@ _None._ - [./src/core/model-prices.generated.json](./src/core/model-prices.generated.json) - [./src/core/paths.ts](./src/core/paths.ts) - [./src/core/prices.ts](./src/core/prices.ts) +- [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/types.ts](./src/core/types.ts) - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json) -- [./src/translationBench/catalog.generated.json](./src/translationBench/catalog.generated.json) -- _…and 29 more under `./src/`._ +- _…and 30 more under `./src/`._ --- -_Auto-generated against commit `2f1ae13a34a138343a5b5113783950a8f1746724` on `2026-08-08T02:25:27.234Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `463e76557daa6dccfa2b5750db60dad5121eb4c3` on `2026-08-12T19:01:55.418Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From edd37b59727c522adb1fca29539fd885d74e4a69 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen <35666615+datduyng@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:58:21 -0500 Subject: [PATCH 03/10] Add translation-bench run config loader (#2856) Load JSON run configs and resolve per-model concurrency from TPM limits. --- .../benchmarks/src/translationBench/index.ts | 1 + .../src/translationBench/runConfig.ts | 223 ++++++++++++++++++ .../test/translationBench.runConfig.spec.ts | 147 ++++++++++++ 3 files changed, 371 insertions(+) create mode 100644 ts/packages/benchmarks/src/translationBench/runConfig.ts create mode 100644 ts/packages/benchmarks/test/translationBench.runConfig.spec.ts diff --git a/ts/packages/benchmarks/src/translationBench/index.ts b/ts/packages/benchmarks/src/translationBench/index.ts index 3d7fec6f80..22cd3309a3 100644 --- a/ts/packages/benchmarks/src/translationBench/index.ts +++ b/ts/packages/benchmarks/src/translationBench/index.ts @@ -2,4 +2,5 @@ // Licensed under the MIT License. export * from "./catalog.js"; +export * from "./runConfig.js"; export * from "./synthesizer/index.js"; diff --git a/ts/packages/benchmarks/src/translationBench/runConfig.ts b/ts/packages/benchmarks/src/translationBench/runConfig.ts new file mode 100644 index 0000000000..c63ed4cd20 --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/runConfig.ts @@ -0,0 +1,223 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { TpmLimits } from "../core/rateLimiter.js"; + +export const DEFAULT_TOK_PER_MIN_PER_SLOT = 70_000; +export const DEFAULT_EST_TOKENS_PER_CALL = 10_400; + +export function defaultRateLimiterDbPath(): string { + return path.join( + os.homedir(), + ".typeagent", + "benchmark", + "rate-limiters", + "tpm.sqlite", + ); +} + +export interface ModelConfig { + tpmLimit?: number; + maxConcurrency?: number; + concurrency?: number; +} + +export interface SynthesizerConfig { + generatorModel?: string; + reviewerModel?: string; + caseCount?: number; + genCases?: number; + maxAttempts?: number; + concurrency?: number; + headroom?: number; +} + +export interface EvalConfig { + models?: string[]; + modelConcurrency?: number; + maxCases?: number | null; + headroom?: number; +} + +export interface BatchConfig { + synthesizer?: SynthesizerConfig; + eval?: EvalConfig; +} + +export interface RunConfigFile { + models?: Record; + base?: BatchConfig; + batches?: Record; +} + +export interface ResolveOptions { + batch?: string; + headroom?: number; + tokPerMinPerSlot?: number; +} + +export interface ResolvedRunConfig { + batch: string; + headroom: number; + generatorModel: string; + reviewerModel: string; + caseCount: number; + genCases: number; + maxAttempts: number; + genConcurrency: number; + evalModels: string[]; + concurrencyByModel: Record; + modelConcurrency: number; + maxCases: number | undefined; + tpmLimits: TpmLimits; +} + +const DEFAULT_BATCH = "eval"; +const DEFAULT_HEADROOM = 0.85; +const DEFAULT_GENERATOR_MODEL = "azure/gpt-5.4"; +const DEFAULT_CASE_COUNT = 1000; +const DEFAULT_GEN_CASES = 2; +const DEFAULT_MAX_ATTEMPTS = 5; +const DEFAULT_GEN_CONCURRENCY = 20; +const DEFAULT_EVAL_CONCURRENCY = 10; + +function isPositive(value: number | undefined): value is number { + return value !== undefined && Number.isFinite(value) && value > 0; +} + +function mergeSection( + base: T | undefined, + override: T | undefined, +): T { + return { ...(base ?? {}), ...(override ?? {}) } as T; +} + +function concurrencyFor( + modelConfig: ModelConfig | undefined, + headroom: number, + tokPerMinPerSlot: number, + fallback: number, +): number { + if (modelConfig === undefined) { + return fallback; + } + if (isPositive(modelConfig.concurrency)) { + return modelConfig.concurrency; + } + if (isPositive(modelConfig.tpmLimit)) { + const derived = Math.max( + 1, + Math.floor((headroom * modelConfig.tpmLimit) / tokPerMinPerSlot), + ); + const cap = isPositive(modelConfig.maxConcurrency) + ? modelConfig.maxConcurrency + : Number.POSITIVE_INFINITY; + return Math.min(derived, cap); + } + return fallback; +} + +export function loadRunConfigFile(filePath: string): RunConfigFile { + if (!fs.existsSync(filePath)) { + return {}; + } + let text: string; + try { + text = fs.readFileSync(filePath, "utf8"); + } catch (error) { + throw new Error( + `runConfig: failed to read ${filePath}: ${String(error)}`, + ); + } + try { + return (JSON.parse(text) as RunConfigFile) ?? {}; + } catch (error) { + throw new Error( + `runConfig: failed to parse ${filePath}: ${String(error)}`, + ); + } +} + +export function resolveRunConfig( + file: RunConfigFile, + options: ResolveOptions = {}, +): ResolvedRunConfig { + const batch = options.batch ?? DEFAULT_BATCH; + const tokPerMinPerSlot = + options.tokPerMinPerSlot ?? DEFAULT_TOK_PER_MIN_PER_SLOT; + + const models = file.models ?? {}; + const base = file.base ?? {}; + if ( + file.batches !== undefined && + Object.keys(file.batches).length > 0 && + !(batch in file.batches) + ) { + throw new Error( + `runConfig: unknown batch '${batch}'. Known batches: ${Object.keys(file.batches).sort().join(", ")}`, + ); + } + const selected = file.batches?.[batch]; + + const synth = mergeSection(base.synthesizer, selected?.synthesizer); + const evalCfg = mergeSection(base.eval, selected?.eval); + + const headroom = + options.headroom ?? + evalCfg.headroom ?? + synth.headroom ?? + DEFAULT_HEADROOM; + + const generatorModel = synth.generatorModel ?? DEFAULT_GENERATOR_MODEL; + const reviewerModel = synth.reviewerModel ?? generatorModel; + + const genConcurrency = concurrencyFor( + models[generatorModel], + headroom, + tokPerMinPerSlot, + synth.concurrency ?? DEFAULT_GEN_CONCURRENCY, + ); + + const evalModels = evalCfg.models ?? []; + const concurrencyByModel: Record = {}; + for (const id of evalModels) { + concurrencyByModel[id] = concurrencyFor( + models[id], + headroom, + tokPerMinPerSlot, + DEFAULT_EVAL_CONCURRENCY, + ); + } + + const tpmLimits: Record = {}; + for (const [id, model] of Object.entries(models)) { + if (isPositive(model.tpmLimit)) { + tpmLimits[id] = model.tpmLimit; + } + } + + return { + batch, + headroom, + generatorModel, + reviewerModel, + caseCount: synth.caseCount ?? DEFAULT_CASE_COUNT, + genCases: synth.genCases ?? DEFAULT_GEN_CASES, + maxAttempts: synth.maxAttempts ?? DEFAULT_MAX_ATTEMPTS, + genConcurrency, + evalModels, + concurrencyByModel, + modelConcurrency: Math.max( + 1, + evalCfg.modelConcurrency ?? evalModels.length, + ), + maxCases: + evalCfg.maxCases === null || evalCfg.maxCases === undefined + ? undefined + : evalCfg.maxCases, + tpmLimits, + }; +} diff --git a/ts/packages/benchmarks/test/translationBench.runConfig.spec.ts b/ts/packages/benchmarks/test/translationBench.runConfig.spec.ts new file mode 100644 index 0000000000..9afbac2e10 --- /dev/null +++ b/ts/packages/benchmarks/test/translationBench.runConfig.spec.ts @@ -0,0 +1,147 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "@jest/globals"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { + loadRunConfigFile, + resolveRunConfig, + type RunConfigFile, +} from "../src/translationBench/runConfig.js"; + +const SAMPLE: RunConfigFile = { + models: { + "azure/gpt-5.4": { tpmLimit: 5_330_000, maxConcurrency: 200 }, + "azure/gpt-4.1": { tpmLimit: 4_850_000, maxConcurrency: 50 }, + "azure/gpt-4.1-mini": { tpmLimit: 15_890_000, maxConcurrency: 200 }, + }, + base: { + synthesizer: { + generatorModel: "azure/gpt-5.4", + reviewerModel: "azure/gpt-5.4", + genCases: 2, + maxAttempts: 5, + }, + eval: { + models: ["azure/gpt-4.1", "azure/gpt-4.1-mini"], + modelConcurrency: 3, + }, + }, + batches: { + eval_fast: { + synthesizer: { caseCount: 100 }, + eval: { maxCases: 100, headroom: 0.9 }, + }, + eval: { + synthesizer: { caseCount: 1000 }, + eval: { maxCases: null, headroom: 0.85 }, + }, + }, +}; + +describe("translationBench runConfig", () => { + it("returns an empty object for a missing file", () => { + expect(loadRunConfigFile("/nonexistent/does-not-exist.json")).toEqual( + {}, + ); + }); + + it("loads and parses a config file from disk", () => { + const dir = mkdtempSync(path.join(tmpdir(), "tb-runconfig-")); + try { + const filePath = path.join(dir, "config.json"); + writeFileSync(filePath, JSON.stringify(SAMPLE)); + expect(loadRunConfigFile(filePath)).toEqual(SAMPLE); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("throws with the file path on malformed json", () => { + const dir = mkdtempSync(path.join(tmpdir(), "tb-runconfig-")); + try { + const filePath = path.join(dir, "bad.json"); + writeFileSync(filePath, "{ not valid json "); + expect(() => loadRunConfigFile(filePath)).toThrow( + /failed to parse/, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("defaults to the eval batch", () => { + const resolved = resolveRunConfig(SAMPLE); + expect(resolved.batch).toBe("eval"); + expect(resolved.caseCount).toBe(1000); + expect(resolved.maxCases).toBeUndefined(); + expect(resolved.headroom).toBe(0.85); + }); + + it("deep-merges the selected batch over base", () => { + const resolved = resolveRunConfig(SAMPLE, { batch: "eval_fast" }); + expect(resolved.caseCount).toBe(100); + expect(resolved.maxCases).toBe(100); + expect(resolved.headroom).toBe(0.9); + expect(resolved.generatorModel).toBe("azure/gpt-5.4"); + expect(resolved.evalModels).toEqual([ + "azure/gpt-4.1", + "azure/gpt-4.1-mini", + ]); + }); + + it("derives per-model concurrency from quota and headroom", () => { + const resolved = resolveRunConfig(SAMPLE, { + batch: "eval", + tokPerMinPerSlot: 70_000, + }); + expect(resolved.concurrencyByModel["azure/gpt-4.1"]).toBe(50); + expect(resolved.concurrencyByModel["azure/gpt-4.1-mini"]).toBe(192); + }); + + it("prefers an explicit model concurrency over derivation", () => { + const file: RunConfigFile = { + models: { + "azure/x": { tpmLimit: 1_000_000, concurrency: 7 }, + }, + base: { eval: { models: ["azure/x"] } }, + batches: { eval: {} }, + }; + const resolved = resolveRunConfig(file); + expect(resolved.concurrencyByModel["azure/x"]).toBe(7); + }); + + it("exposes tpmLimits suitable for the rate limiter", () => { + const resolved = resolveRunConfig(SAMPLE); + expect(resolved.tpmLimits).toEqual({ + "azure/gpt-5.4": 5_330_000, + "azure/gpt-4.1": 4_850_000, + "azure/gpt-4.1-mini": 15_890_000, + }); + }); + + it("omits non-positive tpmLimits", () => { + const file: RunConfigFile = { + models: { + "azure/on": { tpmLimit: 1_000_000 }, + "azure/off": { tpmLimit: 0 }, + }, + }; + const resolved = resolveRunConfig(file); + expect(resolved.tpmLimits).toEqual({ "azure/on": 1_000_000 }); + }); + + it("applies built-in defaults for an empty config", () => { + const resolved = resolveRunConfig({}); + expect(resolved.generatorModel).toBe("azure/gpt-5.4"); + expect(resolved.reviewerModel).toBe("azure/gpt-5.4"); + expect(resolved.caseCount).toBe(1000); + expect(resolved.genCases).toBe(2); + expect(resolved.evalModels).toEqual([]); + expect(resolved.tpmLimits).toEqual({}); + expect(resolved.modelConcurrency).toBe(1); + }); +}); From f8f35383c735cec3d8b9445a1777a00e6992f592 Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Wed, 12 Aug 2026 21:08:13 +0000 Subject: [PATCH 04/10] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index d67aa95841..4cadf954dc 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -52,10 +52,10 @@ _None._ - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/types.ts](./src/core/types.ts) - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json) -- _…and 30 more under `./src/`._ +- _…and 31 more under `./src/`._ --- -_Auto-generated against commit `463e76557daa6dccfa2b5750db60dad5121eb4c3` on `2026-08-12T19:01:55.418Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `edd37b59727c522adb1fca29539fd885d74e4a69` on `2026-08-12T21:05:45.079Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From 8b55a7d54f478dabdf0b721460fa5c7121727ac7 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 12 Aug 2026 14:26:32 -0700 Subject: [PATCH 05/10] refactor(benchmarks): split resolveRunConfig to pass complexity cap - Extract batch lookup, TPM map, and eval concurrency helpers - Keep resolveRunConfig under the new-file cyclomatic cap of 25 --- .../src/translationBench/runConfig.ts | 109 +++++++++++------- 1 file changed, 68 insertions(+), 41 deletions(-) diff --git a/ts/packages/benchmarks/src/translationBench/runConfig.ts b/ts/packages/benchmarks/src/translationBench/runConfig.ts index c63ed4cd20..84a3cf88b3 100644 --- a/ts/packages/benchmarks/src/translationBench/runConfig.ts +++ b/ts/packages/benchmarks/src/translationBench/runConfig.ts @@ -141,6 +141,59 @@ export function loadRunConfigFile(filePath: string): RunConfigFile { } } +function selectedBatch( + batches: Record | undefined, + batch: string, +): BatchConfig | undefined { + if (batches !== undefined && Object.keys(batches).length > 0) { + if (!(batch in batches)) { + throw new Error( + `runConfig: unknown batch '${batch}'. Known batches: ${Object.keys(batches).sort().join(", ")}`, + ); + } + } + return batches?.[batch]; +} + +function tpmLimitsFromModels( + models: Record, +): Record { + const tpmLimits: Record = {}; + for (const [id, model] of Object.entries(models)) { + if (isPositive(model.tpmLimit)) { + tpmLimits[id] = model.tpmLimit; + } + } + return tpmLimits; +} + +function evalConcurrencyByModel( + evalModels: string[], + models: Record, + headroom: number, + tokPerMinPerSlot: number, +): Record { + const concurrencyByModel: Record = {}; + for (const id of evalModels) { + concurrencyByModel[id] = concurrencyFor( + models[id], + headroom, + tokPerMinPerSlot, + DEFAULT_EVAL_CONCURRENCY, + ); + } + return concurrencyByModel; +} + +function optionalMaxCases( + maxCases: number | null | undefined, +): number | undefined { + if (maxCases === null || maxCases === undefined) { + return undefined; + } + return maxCases; +} + export function resolveRunConfig( file: RunConfigFile, options: ResolveOptions = {}, @@ -151,16 +204,7 @@ export function resolveRunConfig( const models = file.models ?? {}; const base = file.base ?? {}; - if ( - file.batches !== undefined && - Object.keys(file.batches).length > 0 && - !(batch in file.batches) - ) { - throw new Error( - `runConfig: unknown batch '${batch}'. Known batches: ${Object.keys(file.batches).sort().join(", ")}`, - ); - } - const selected = file.batches?.[batch]; + const selected = selectedBatch(file.batches, batch); const synth = mergeSection(base.synthesizer, selected?.synthesizer); const evalCfg = mergeSection(base.eval, selected?.eval); @@ -173,31 +217,7 @@ export function resolveRunConfig( const generatorModel = synth.generatorModel ?? DEFAULT_GENERATOR_MODEL; const reviewerModel = synth.reviewerModel ?? generatorModel; - - const genConcurrency = concurrencyFor( - models[generatorModel], - headroom, - tokPerMinPerSlot, - synth.concurrency ?? DEFAULT_GEN_CONCURRENCY, - ); - const evalModels = evalCfg.models ?? []; - const concurrencyByModel: Record = {}; - for (const id of evalModels) { - concurrencyByModel[id] = concurrencyFor( - models[id], - headroom, - tokPerMinPerSlot, - DEFAULT_EVAL_CONCURRENCY, - ); - } - - const tpmLimits: Record = {}; - for (const [id, model] of Object.entries(models)) { - if (isPositive(model.tpmLimit)) { - tpmLimits[id] = model.tpmLimit; - } - } return { batch, @@ -207,17 +227,24 @@ export function resolveRunConfig( caseCount: synth.caseCount ?? DEFAULT_CASE_COUNT, genCases: synth.genCases ?? DEFAULT_GEN_CASES, maxAttempts: synth.maxAttempts ?? DEFAULT_MAX_ATTEMPTS, - genConcurrency, + genConcurrency: concurrencyFor( + models[generatorModel], + headroom, + tokPerMinPerSlot, + synth.concurrency ?? DEFAULT_GEN_CONCURRENCY, + ), evalModels, - concurrencyByModel, + concurrencyByModel: evalConcurrencyByModel( + evalModels, + models, + headroom, + tokPerMinPerSlot, + ), modelConcurrency: Math.max( 1, evalCfg.modelConcurrency ?? evalModels.length, ), - maxCases: - evalCfg.maxCases === null || evalCfg.maxCases === undefined - ? undefined - : evalCfg.maxCases, - tpmLimits, + maxCases: optionalMaxCases(evalCfg.maxCases), + tpmLimits: tpmLimitsFromModels(models), }; } From 8e859effc6196a03d5fdeca8ad65737db7431d02 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 12 Aug 2026 15:02:41 -0700 Subject: [PATCH 06/10] chore: requeue CI after flaky infra From 718a80a73e0e5ae91dfadac7288b6e4b222cffd0 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 12 Aug 2026 15:13:01 -0700 Subject: [PATCH 07/10] chore: requeue CI after flaky keytar install From 7fa6e3e1421c88fb28cb3f430fa989d74b473a0a Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Wed, 12 Aug 2026 23:30:49 +0000 Subject: [PATCH 08/10] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 4cadf954dc..0cbe8d7a73 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -52,10 +52,10 @@ _None._ - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/types.ts](./src/core/types.ts) - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json) -- _…and 31 more under `./src/`._ +- _…and 33 more under `./src/`._ --- -_Auto-generated against commit `edd37b59727c522adb1fca29539fd885d74e4a69` on `2026-08-12T21:05:45.079Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `c06920299b16f60235d8a2d13eb58c59ea555994` on `2026-08-12T23:28:20.828Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From 1bcb4efb63db8d2106149654cd3018cb0d154ccc Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Thu, 13 Aug 2026 01:29:01 +0000 Subject: [PATCH 09/10] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 0cbe8d7a73..1ce9e2b2f8 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -52,10 +52,10 @@ _None._ - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/types.ts](./src/core/types.ts) - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json) -- _…and 33 more under `./src/`._ +- _…and 34 more under `./src/`._ --- -_Auto-generated against commit `c06920299b16f60235d8a2d13eb58c59ea555994` on `2026-08-12T23:28:20.828Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `d9cf714f7d151120013855a722e7583cbf2c30d7` on `2026-08-13T01:26:47.704Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From a8d834b27f932ef154a7ffc32ab38db0bf41ab1e Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 12 Aug 2026 18:55:26 -0700 Subject: [PATCH 10/10] chore: re-trigger CI for #2855