Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/data.md
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ Test-only exports `_resetCache()` and `_cacheSize()` allow test isolation.

The optional `filter` callback enables pre-filtering JSONL rows (used by `test-run` to keep only `type === 'result'` events). Old all-events schema is detected by the presence of a `message` column and auto-migrated by deleting and rebuilding.

`updateAllParquet()` orchestrates import for eight tables: `test-config`, `test-run`, `test-results`, `output-files`, `scil-iteration`, `scil-summary`, `acil-iteration`, and `acil-summary`. SCIL and ACIL summaries each require a special step that converts per-run `.json` files to a temp JSONL before import.
`updateAllParquet()` orchestrates import for eight tables: `test-config`, `test-run`, `test-results`, `output-files`, `scil-iteration`, `scil-summary`, `acil-iteration`, and `acil-summary`. SCIL and ACIL summaries each require a special step that converts per-run `.json` files to a temp JSONL before import. It creates the data directory first if it does not exist, since DuckDB's `COPY ... TO` will not create parent directories.

### Analytics Queries (`analytics.ts`, `run-status.ts`)

Expand Down
11 changes: 11 additions & 0 deletions packages/data/src/analytics.integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,17 @@ describe('updateAllParquet', () => {
expect(rows.every((r) => r.type === 'result')).toBe(true)
})

it('creates the data directory when it does not exist yet', async () => {
const outputDir = path.join(tmpDir, 'output')
const dataDir = path.join(tmpDir, 'analytics')
await writeRunFixture({ outputDir, testRunId: '20260101T100001', eval: 's', testName: 't' })

const { updated } = await updateAllParquet({ outputDir, dataDir })

expect(updated).toEqual(expect.arrayContaining(['test-config', 'test-run', 'test-results']))
expect(existsSync(path.join(dataDir, 'test-config.parquet'))).toBe(true)
})

it('returns empty updated list when no JSONL files exist', async () => {
const outputDir = path.join(tmpDir, 'output-empty')
const dataDir = path.join(tmpDir, 'analytics')
Expand Down
4 changes: 3 additions & 1 deletion packages/data/src/analytics.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import crypto from 'node:crypto'
import { existsSync } from 'node:fs'
import { readdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'
import { mkdir, readdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { type DuckDBConnection, DuckDBInstance } from '@duckdb/node-api'
Expand Down Expand Up @@ -165,6 +165,8 @@ export async function updateAllParquet({
{ name: 'output-files', glob: `${outputDir}/*/output-files.jsonl`, parquet: `${dataDir}/output-files.parquet` },
]

await mkdir(dataDir, { recursive: true })

const updated: string[] = []

for (const table of tables) {
Expand Down
Loading