Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 41 additions & 1 deletion packages/data/src/analytics.integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1077,7 +1077,47 @@ describe('infrastructure-error filtering', () => {
// The infrastructure-error record should still appear in llmJudgeGroups
expect(details.llmJudgeGroups).toHaveLength(1)
expect(details.llmJudgeGroups[0].passed).toBe(false)
expect(Number(details.llmJudgeGroups[0].score)).toBe(0)
expect(details.llmJudgeGroups[0].score).toBe(0)
})
})

describe('queryTestRunDetails (whole-number judge values)', () => {
it('returns judge score and threshold as JSON-serializable numbers', async () => {
const outputDir = path.join(tmpDir, 'output')
const dataDir = path.join(tmpDir, 'analytics')

const runId = '20260101T100004'
const runDir = path.join(outputDir, runId)

await writeJsonl(path.join(runDir, 'test-config.jsonl'), [
makeConfigRecord({ testRunId: runId, eval: 's', testName: 'my test' }),
])
await writeJsonl(path.join(runDir, 'test-run.jsonl'), [
makeRunResultRecord({ testRunId: runId, eval: 's', testName: 'my test' }),
])
// whole numbers are written to JSONL without a decimal point, so DuckDB infers BIGINT
await writeJsonl(path.join(runDir, 'test-results.jsonl'), [
makeResultRecord({
testRunId: runId,
eval: 's',
testName: 'my test',
expectType: 'llm-judge-aggregate',
expectValue: 'rubric.md',
passed: true,
judgeModel: 'opus',
judgeThreshold: 1,
judgeScore: 1,
rubricFile: 'rubric.md',
}),
])

await updateAllParquet({ outputDir, dataDir })
const details = await queryTestRunDetails(dataDir, runId)

expect(details.llmJudgeGroups).toHaveLength(1)
expect(details.llmJudgeGroups[0].score).toBe(1)
expect(details.llmJudgeGroups[0].threshold).toBe(1)
expect(() => JSON.stringify(details)).not.toThrow()
})
})

Expand Down
6 changes: 4 additions & 2 deletions packages/data/src/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -527,8 +527,10 @@ export async function queryTestRunDetails(dataDir: string, testRunId: string): P
testName,
rubricFile: agg.rubric_file!,
model: agg.judge_model ?? 'unknown',
threshold: agg.judge_threshold ?? 1.0,
score: agg.judge_score ?? 0,
// DuckDB infers BIGINT for whole-number JSON values (e.g. a perfect score of 1),
// which returns a BigInt that JSON serialization rejects — coerce to number
threshold: Number(agg.judge_threshold ?? 1.0),
score: Number(agg.judge_score ?? 0),
passed: agg.passed,
resultText: resultTextByTest.get(testName),
criteria: group.criteria,
Expand Down
Loading