diff --git a/docs/data.md b/docs/data.md index 7523f8b..38f20e8 100644 --- a/docs/data.md +++ b/docs/data.md @@ -206,7 +206,7 @@ Test-only exports `_resetCache()` and `_cacheSize()` allow test isolation. The optional `filter` callback enables pre-filtering JSONL rows (used by `test-run` to keep only `type === 'result'` events). Old all-events schema is detected by the presence of a `message` column and auto-migrated by deleting and rebuilding. -`updateAllParquet()` orchestrates import for eight tables: `test-config`, `test-run`, `test-results`, `output-files`, `scil-iteration`, `scil-summary`, `acil-iteration`, and `acil-summary`. SCIL and ACIL summaries each require a special step that converts per-run `.json` files to a temp JSONL before import. +`updateAllParquet()` orchestrates import for eight tables: `test-config`, `test-run`, `test-results`, `output-files`, `scil-iteration`, `scil-summary`, `acil-iteration`, and `acil-summary`. SCIL and ACIL summaries each require a special step that converts per-run `.json` files to a temp JSONL before import. It creates the data directory first if it does not exist, since DuckDB's `COPY ... TO` will not create parent directories. ### Analytics Queries (`analytics.ts`, `run-status.ts`) diff --git a/packages/data/src/analytics.integration.test.ts b/packages/data/src/analytics.integration.test.ts index 47d019b..e659d57 100644 --- a/packages/data/src/analytics.integration.test.ts +++ b/packages/data/src/analytics.integration.test.ts @@ -205,6 +205,17 @@ describe('updateAllParquet', () => { expect(rows.every((r) => r.type === 'result')).toBe(true) }) + it('creates the data directory when it does not exist yet', async () => { + const outputDir = path.join(tmpDir, 'output') + const dataDir = path.join(tmpDir, 'analytics') + await writeRunFixture({ outputDir, testRunId: '20260101T100001', eval: 's', testName: 't' }) + + const { updated } = await updateAllParquet({ outputDir, dataDir }) + + expect(updated).toEqual(expect.arrayContaining(['test-config', 'test-run', 'test-results'])) + expect(existsSync(path.join(dataDir, 'test-config.parquet'))).toBe(true) + }) + it('returns empty updated list when no JSONL files exist', async () => { const outputDir = path.join(tmpDir, 'output-empty') const dataDir = path.join(tmpDir, 'analytics') diff --git a/packages/data/src/analytics.ts b/packages/data/src/analytics.ts index c742ce8..add6dca 100644 --- a/packages/data/src/analytics.ts +++ b/packages/data/src/analytics.ts @@ -1,6 +1,6 @@ import crypto from 'node:crypto' import { existsSync } from 'node:fs' -import { readdir, readFile, rename, unlink, writeFile } from 'node:fs/promises' +import { mkdir, readdir, readFile, rename, unlink, writeFile } from 'node:fs/promises' import os from 'node:os' import path from 'node:path' import { type DuckDBConnection, DuckDBInstance } from '@duckdb/node-api' @@ -165,6 +165,8 @@ export async function updateAllParquet({ { name: 'output-files', glob: `${outputDir}/*/output-files.jsonl`, parquet: `${dataDir}/output-files.parquet` }, ] + await mkdir(dataDir, { recursive: true }) + const updated: string[] = [] for (const table of tables) {