From f1e84cb088380d5b6b9e20a61d25924e81737877 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 15 Apr 2026 17:11:25 +0200 Subject: [PATCH] chore: add backend preview validation to ai evals (#8827) * feat: add backend preview validation to ai evals Co-Authored-By: Claude Opus 4.5 * fix: refresh shared preview workspace assets Co-Authored-By: Claude Opus 4.5 * fix: harden shared backend preview validation Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- ai_evals/README.md | 21 + .../adapters/frontend/backendPreview.test.ts | 246 +++++++++ ai_evals/adapters/frontend/backendPreview.ts | 502 ++++++++++++++++++ ai_evals/adapters/frontend/benchmarkRunner.ts | 14 +- ai_evals/adapters/frontend/runtime.ts | 4 +- ai_evals/cases/flow.yaml | 15 + ai_evals/cli/index.ts | 19 + ai_evals/core/backendValidation.test.ts | 36 ++ ai_evals/core/backendValidation.ts | 104 ++++ ai_evals/core/cases.test.ts | 18 + ai_evals/core/cases.ts | 4 +- ai_evals/core/runSuite.ts | 39 +- ai_evals/core/types.ts | 24 + .../expected/test1_reuse_existing_script.json | 2 +- .../test1_reuse_existing_script_initial.json | 2 +- ai_evals/modes/flow.ts | 93 +++- ai_evals/modes/script.ts | 64 ++- 17 files changed, 1196 insertions(+), 11 deletions(-) create mode 100644 ai_evals/adapters/frontend/backendPreview.test.ts create mode 100644 ai_evals/adapters/frontend/backendPreview.ts create mode 100644 ai_evals/core/backendValidation.test.ts create mode 100644 ai_evals/core/backendValidation.ts create mode 100644 ai_evals/core/cases.test.ts diff --git a/ai_evals/README.md b/ai_evals/README.md index 353bee9dc3..2136d68151 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -55,6 +55,7 @@ bun run cli -- run flow flow-test4-order-processing-loop --model opus bun run cli -- run flow flow-test0-sum-two-numbers --models haiku,opus,4o bun run cli -- run flow flow-test0-sum-two-numbers --runs 3 --verbose bun run cli -- run flow --record +WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000 bun run cli -- run flow --backend-validation preview bun run cli -- run cli bun-hello-script ``` @@ -72,6 +73,7 @@ Public CLI surface: - `--models `: run the same cases sequentially against several model aliases - `--verbose`: stream assistant output for frontend runs - `--record`: append a compact tracked summary line to `ai_evals/history/.jsonl` for full-suite runs only +- `--backend-validation `: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals ## Models @@ -114,6 +116,7 @@ Optional fields: - `initial`: starting state fixture - `expected`: expected artifact fixture - `validate`: extra deterministic validation rules +- `runtime.backendPreview`: optional real backend preview config for smoke validation For `flow` mode, `validate` can express requirements such as: @@ -125,6 +128,23 @@ For `flow` mode, an `initial` fixture can also include a benchmark workspace cat existing scripts and flows. That lets the real `search_workspace` and `get_runnable_details` tools discover reusable workspace runnables during evals. +If `--backend-validation preview` is enabled: + +- `script` evals run a real backend script preview in an isolated temp workspace +- `flow` evals run a real backend flow preview only for cases that define `runtime.backendPreview` +- `flow` cases with `initial.workspace` fixtures seed those scripts and flows into the preview workspace before preview +- when `WMILL_AI_EVAL_BACKEND_WORKSPACE` is set, `ai_evals` treats that workspace as a dedicated test workspace, clears managed eval assets under `f/evals/*` before each preview run, and then reseeds the current case fixtures + +Supported backend validation env vars: + +- `WMILL_AI_EVAL_BACKEND_VALIDATION=preview` +- `WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000` +- `WMILL_AI_EVAL_BACKEND_EMAIL=admin@windmill.dev` +- `WMILL_AI_EVAL_BACKEND_PASSWORD=changeme` +- `WMILL_AI_EVAL_BACKEND_WORKSPACE=integration-tests` to reuse an existing workspace on CE installs with low workspace limits +- `WMILL_AI_EVAL_KEEP_WORKSPACES=1` +- `WMILL_AI_EVAL_WORKSPACE_PREFIX=ai-evals` + ## Results And Artifacts Every run writes: @@ -158,6 +178,7 @@ Typical artifacts by mode: - `script`: `script.json` plus the generated script file - `app`: `app.json` plus frontend/backend files - `cli`: `assistant-output.txt` plus generated workspace files +- backend-validated attempts also include `backend-preview.json` ## Layout diff --git a/ai_evals/adapters/frontend/backendPreview.test.ts b/ai_evals/adapters/frontend/backendPreview.test.ts new file mode 100644 index 0000000000..2f12c9a896 --- /dev/null +++ b/ai_evals/adapters/frontend/backendPreview.test.ts @@ -0,0 +1,246 @@ +import { afterEach, describe, expect, it } from 'bun:test' +import type { BackendValidationSettings } from '../../core/backendValidation' +import { BackendPreviewClient } from './backendPreview' + +const ORIGINAL_FETCH = globalThis.fetch + +afterEach(() => { + globalThis.fetch = ORIGINAL_FETCH +}) + +describe('BackendPreviewClient', () => { + it('updates an existing seeded script on path conflict and waits for deployment', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + globalThis.fetch = mockFetch( + requests, + textResponse(200, 'token'), + textResponse(200, ''), + textResponse(400, 'Path conflict for f/evals/add_two_numbers with non-archived hash 123'), + jsonResponse(200, { hash: '123' }), + textResponse(200, '456'), + jsonResponse(200, { lock: 'script.lock', lock_error_logs: null }) + ) + + const client = new BackendPreviewClient( + buildSettings({ baseUrl: 'http://backend.test/script-upsert' }) + ) + + await client.createScript({ + workspaceId: 'test', + path: 'f/evals/add_two_numbers', + summary: 'Add two numbers', + content: 'export async function main(a: number, b: number) { return a + b }', + language: 'bun' + }) + + expect(requests.map((entry) => entry.url)).toEqual([ + 'http://backend.test/script-upsert/api/auth/login', + 'http://backend.test/script-upsert/api/w/test/folders/create', + 'http://backend.test/script-upsert/api/w/test/scripts/create', + 'http://backend.test/script-upsert/api/w/test/scripts/get/p/f/evals/add_two_numbers', + 'http://backend.test/script-upsert/api/w/test/scripts/create', + 'http://backend.test/script-upsert/api/w/test/scripts/deployment_status/h/456' + ]) + + const updateRequest = requests[4] + expect(updateRequest.init?.method).toBe('POST') + expect(JSON.parse(String(updateRequest.init?.body))).toMatchObject({ + path: 'f/evals/add_two_numbers', + parent_hash: '123', + language: 'bun' + }) + }) + + it('updates an existing seeded flow on create conflict', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + globalThis.fetch = mockFetch( + requests, + textResponse(200, 'token'), + textResponse(200, ''), + textResponse(400, 'Flow f/evals/add_numbers_flow already exists'), + textResponse(200, '') + ) + + const client = new BackendPreviewClient( + buildSettings({ baseUrl: 'http://backend.test/flow-upsert' }) + ) + + await client.createFlow({ + workspaceId: 'test', + path: 'f/evals/add_numbers_flow', + summary: 'Add numbers', + value: { modules: [] } + }) + + expect(requests.map((entry) => entry.url)).toEqual([ + 'http://backend.test/flow-upsert/api/auth/login', + 'http://backend.test/flow-upsert/api/w/test/folders/create', + 'http://backend.test/flow-upsert/api/w/test/flows/create', + 'http://backend.test/flow-upsert/api/w/test/flows/update/f/evals/add_numbers_flow' + ]) + + const updateRequest = requests[3] + expect(updateRequest.init?.method).toBe('POST') + expect(JSON.parse(String(updateRequest.init?.body))).toMatchObject({ + path: 'f/evals/add_numbers_flow', + value: { modules: [] } + }) + }) + + it('serializes shared-workspace validations inside the overridden workspace', async () => { + globalThis.fetch = async (input) => { + const url = String(input) + if (url.endsWith('/api/auth/login')) { + return textResponse(200, 'token') + } + if (url.endsWith('/api/workspaces/exists')) { + return textResponse(200, 'true') + } + if (url.endsWith('/api/w/shared-preview/flows/list_paths')) { + return jsonResponse(200, []) + } + if (url.endsWith('/api/w/shared-preview/scripts/list_paths')) { + return jsonResponse(200, []) + } + throw new Error(`Unexpected fetch: ${url}`) + } + + const client = new BackendPreviewClient( + buildSettings({ + baseUrl: 'http://backend.test/shared-lock', + workspaceOverride: 'shared-preview' + }) + ) + + const order: string[] = [] + let releaseFirst: (() => void) | undefined + let notifyFirstStart: (() => void) | undefined + const firstStarted = new Promise((resolve) => { + notifyFirstStart = resolve + }) + + const first = client.withWorkspace('flow-test1', 1, async () => { + order.push('first:start') + notifyFirstStart?.() + await new Promise((resolve) => { + releaseFirst = resolve + }) + order.push('first:end') + }) + + const second = client.withWorkspace('flow-test2', 1, async () => { + order.push('second:start') + order.push('second:end') + }) + + await firstStarted + expect(order).toEqual(['first:start']) + + releaseFirst?.() + await Promise.all([first, second]) + + expect(order).toEqual(['first:start', 'first:end', 'second:start', 'second:end']) + }) + + it('clears managed shared-workspace assets before preview runs', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + globalThis.fetch = mockFetch( + requests, + textResponse(200, 'token'), + textResponse(200, 'true'), + jsonResponse(200, ['f/evals/old_subflow', 'u/admin/keep_flow']), + textResponse(200, ''), + jsonResponse(200, ['f/evals/old_script', 'f/shared/keep_script']), + textResponse(200, '') + ) + + const client = new BackendPreviewClient( + buildSettings({ + baseUrl: 'http://backend.test/shared-cleanup', + workspaceOverride: 'shared-preview' + }) + ) + + await client.withWorkspace('flow-test1', 1, async () => undefined) + + expect(requests.map((entry) => entry.url)).toEqual([ + 'http://backend.test/shared-cleanup/api/auth/login', + 'http://backend.test/shared-cleanup/api/workspaces/exists', + 'http://backend.test/shared-cleanup/api/w/shared-preview/flows/list_paths', + 'http://backend.test/shared-cleanup/api/w/shared-preview/flows/delete/f/evals/old_subflow', + 'http://backend.test/shared-cleanup/api/w/shared-preview/scripts/list_paths', + 'http://backend.test/shared-cleanup/api/w/shared-preview/scripts/delete/p/f/evals/old_script' + ]) + }) + + it('retries login after a cached login failure', async () => { + const requests: Array<{ url: string; init?: RequestInit }> = [] + globalThis.fetch = mockFetch( + requests, + textResponse(503, 'backend starting'), + textResponse(200, 'token'), + textResponse(200, 'true'), + jsonResponse(200, []), + jsonResponse(200, []) + ) + + const client = new BackendPreviewClient( + buildSettings({ + baseUrl: 'http://backend.test/login-retry', + workspaceOverride: 'shared-preview' + }) + ) + + await expect(client.withWorkspace('flow-test1', 1, async () => undefined)).rejects.toThrow( + 'login for backend validation failed' + ) + await expect(client.withWorkspace('flow-test1', 1, async () => 'ok')).resolves.toBe('ok') + + expect( + requests.filter((entry) => entry.url === 'http://backend.test/login-retry/api/auth/login') + ).toHaveLength(2) + }) +}) + +function buildSettings( + overrides: Partial = {} +): BackendValidationSettings { + return { + mode: 'preview', + baseUrl: 'http://backend.test/default', + email: 'admin@windmill.dev', + password: 'changeme', + keepWorkspaces: true, + workspacePrefix: 'ai-evals', + pollIntervalMs: 1, + maxWaitMs: 50, + ...overrides + } +} + +function mockFetch( + requests: Array<{ url: string; init?: RequestInit }>, + ...responses: Response[] +): typeof fetch { + const queue = [...responses] + return async (input, init) => { + const url = String(input) + requests.push({ url, init }) + const next = queue.shift() + if (!next) { + throw new Error(`Unexpected fetch: ${url}`) + } + return next + } +} + +function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' } + }) +} + +function textResponse(status: number, body: string): Response { + return new Response(body, { status }) +} diff --git a/ai_evals/adapters/frontend/backendPreview.ts b/ai_evals/adapters/frontend/backendPreview.ts new file mode 100644 index 0000000000..e1be934564 --- /dev/null +++ b/ai_evals/adapters/frontend/backendPreview.ts @@ -0,0 +1,502 @@ +import { randomUUID } from 'node:crypto' +import type { BackendValidationSettings } from '../../core/backendValidation' + +interface CompletedJobResultMaybe { + completed: boolean + result: unknown + success?: boolean + started?: boolean +} + +interface ScriptDeploymentStatus { + lock?: unknown + lock_error_logs?: string | null +} + +export interface CompletedPreviewJob { + id: string + success: boolean + result: unknown + logs?: string | null + raw: Record +} + +const tokenCache = new Map>() +const sharedWorkspaceQueue = new Map>() +const managedSharedWorkspacePrefixes = ['f/evals/'] + +export class BackendPreviewClient { + constructor(private readonly settings: BackendValidationSettings) {} + + async withWorkspace( + caseId: string, + attempt: number, + body: (workspaceId: string) => Promise + ): Promise { + const workspaceId = + this.settings.workspaceOverride ?? + buildWorkspaceId(this.settings.workspacePrefix, caseId, attempt) + + const run = async () => { + await this.ensureWorkspace(workspaceId) + if (this.settings.workspaceOverride) { + await this.clearManagedSharedWorkspaceAssets(workspaceId) + } + + try { + return await body(workspaceId) + } finally { + if (!this.settings.keepWorkspaces && !this.settings.workspaceOverride) { + await this.deleteWorkspace(workspaceId).catch(() => undefined) + } + } + } + + if (this.settings.workspaceOverride) { + return await withSharedWorkspaceLock(workspaceId, run) + } + + return await run() + } + + async createScript(input: { + workspaceId: string + path: string + summary: string + description?: string + schema?: Record + content: string + language: string + }): Promise { + await this.ensureFolderForPath(input.workspaceId, input.path) + + const payload = { + path: input.path, + summary: input.summary, + description: input.description ?? '', + content: input.content, + schema: input.schema ?? { type: 'object', properties: {}, required: [] }, + is_template: false, + language: input.language, + kind: 'script' + } + + const response = await this.request(`/w/${encodeURIComponent(input.workspaceId)}/scripts/create`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }) + + if (response.ok) { + await this.waitForScriptDeployment(input.workspaceId, input.path, (await response.text()).trim()) + return + } + + const message = await response.text() + if (!isConflictMessage(message)) { + throw new Error(`create script ${input.path} failed: ${response.status} ${response.statusText} - ${message}`) + } + + const currentScript = await this.getScriptByPath(input.workspaceId, input.path) + const currentHash = readStringField(currentScript, 'hash', `script ${input.path}`) + const updateResponse = await this.request( + `/w/${encodeURIComponent(input.workspaceId)}/scripts/create`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + ...payload, + parent_hash: currentHash + }) + } + ) + await expectOk(updateResponse, `update script ${input.path}`) + await this.waitForScriptDeployment(input.workspaceId, input.path, (await updateResponse.text()).trim()) + } + + async createFlow(input: { + workspaceId: string + path: string + summary: string + description?: string + schema?: Record + value: Record + }): Promise { + await this.ensureFolderForPath(input.workspaceId, input.path) + + const payload = { + path: input.path, + summary: input.summary, + description: input.description ?? '', + schema: input.schema ?? { type: 'object', properties: {}, required: [] }, + value: input.value + } + + const response = await this.request(`/w/${encodeURIComponent(input.workspaceId)}/flows/create`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }) + + if (response.ok) { + return + } + + const message = await response.text() + if (!isConflictMessage(message)) { + throw new Error(`create flow ${input.path} failed: ${response.status} ${response.statusText} - ${message}`) + } + + const updateResponse = await this.request( + `/w/${encodeURIComponent(input.workspaceId)}/flows/update/${input.path}`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + } + ) + await expectOk(updateResponse, `update flow ${input.path}`) + } + + async runScriptPreview(input: { + workspaceId: string + content: string + args: Record + language: string + path?: string + timeoutSeconds?: number + }): Promise { + const response = await this.request( + withQuery(`/w/${encodeURIComponent(input.workspaceId)}/jobs/run/preview`, { + timeout: input.timeoutSeconds + }), + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + content: input.content, + args: input.args, + language: input.language, + path: input.path + }) + } + ) + + await expectOk(response, 'start script preview') + const jobId = (await response.text()).trim() + return await this.waitForCompletedJob(input.workspaceId, jobId) + } + + async runFlowPreview(input: { + workspaceId: string + value: Record + args: Record + timeoutSeconds?: number + path?: string + }): Promise { + const response = await this.request( + withQuery(`/w/${encodeURIComponent(input.workspaceId)}/jobs/run/preview_flow`, { + timeout: input.timeoutSeconds + }), + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + value: input.value, + args: input.args, + path: input.path + }) + } + ) + + await expectOk(response, 'start flow preview') + const jobId = (await response.text()).trim() + return await this.waitForCompletedJob(input.workspaceId, jobId) + } + + private async ensureWorkspace(workspaceId: string): Promise { + const existsResponse = await this.request('/workspaces/exists', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ id: workspaceId }) + }) + await expectOk(existsResponse, `check workspace ${workspaceId}`) + + if ((await existsResponse.text()).trim() === 'true') { + return + } + + const createResponse = await this.request('/workspaces/create', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ id: workspaceId, name: workspaceId }) + }) + try { + await expectOk(createResponse, `create workspace ${workspaceId}`) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + if (message.includes('maximum number of workspaces')) { + throw new Error( + `${message}. Reuse an existing workspace with WMILL_AI_EVAL_BACKEND_WORKSPACE=.` + ) + } + throw error + } + } + + private async deleteWorkspace(workspaceId: string): Promise { + const response = await this.request(`/workspaces/delete/${encodeURIComponent(workspaceId)}`, { + method: 'DELETE' + }) + await expectOk(response, `delete workspace ${workspaceId}`) + } + + private async ensureFolderForPath(workspaceId: string, path: string): Promise { + const folderName = extractFolderName(path) + if (!folderName) { + return + } + + const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/folders/create`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: folderName }) + }) + + if (response.ok) { + return + } + + const message = await response.text() + if (!message.toLowerCase().includes('already exists')) { + throw new Error(`Failed to create folder ${folderName}: ${message}`) + } + } + + private async waitForCompletedJob( + workspaceId: string, + jobId: string + ): Promise { + const deadline = Date.now() + this.settings.maxWaitMs + + while (Date.now() < deadline) { + const maybeResponse = await this.request( + `/w/${encodeURIComponent(workspaceId)}/jobs_u/completed/get_result_maybe/${encodeURIComponent(jobId)}?get_started=false` + ) + await expectOk(maybeResponse, `poll job ${jobId}`) + const maybeResult = (await maybeResponse.json()) as CompletedJobResultMaybe + + if (maybeResult.completed) { + const completedResponse = await this.request( + `/w/${encodeURIComponent(workspaceId)}/jobs_u/completed/get/${encodeURIComponent(jobId)}` + ) + await expectOk(completedResponse, `get completed job ${jobId}`) + const completedJob = (await completedResponse.json()) as Record + return { + id: jobId, + success: Boolean(maybeResult.success), + result: maybeResult.result, + logs: + typeof completedJob.logs === 'string' || completedJob.logs === null + ? (completedJob.logs as string | null) + : null, + raw: completedJob + } + } + + await new Promise((resolve) => setTimeout(resolve, this.settings.pollIntervalMs)) + } + + throw new Error(`Timed out waiting for preview job ${jobId} to complete`) + } + + private async getScriptByPath(workspaceId: string, path: string): Promise> { + const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/scripts/get/p/${path}`) + await expectOk(response, `get script ${path}`) + return (await response.json()) as Record + } + + private async clearManagedSharedWorkspaceAssets(workspaceId: string): Promise { + const flowPaths = await this.listFlowPaths(workspaceId) + for (const path of flowPaths.filter(isManagedSharedWorkspacePath)) { + await this.deleteFlowByPath(workspaceId, path) + } + + const scriptPaths = await this.listScriptPaths(workspaceId) + for (const path of scriptPaths.filter(isManagedSharedWorkspacePath)) { + await this.deleteScriptByPath(workspaceId, path) + } + } + + private async listFlowPaths(workspaceId: string): Promise { + const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/flows/list_paths`) + await expectOk(response, `list flows in workspace ${workspaceId}`) + return await response.json() + } + + private async listScriptPaths(workspaceId: string): Promise { + const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/scripts/list_paths`) + await expectOk(response, `list scripts in workspace ${workspaceId}`) + return await response.json() + } + + private async deleteFlowByPath(workspaceId: string, path: string): Promise { + const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/flows/delete/${path}`, { + method: 'DELETE' + }) + await expectOk(response, `delete flow ${path}`) + } + + private async deleteScriptByPath(workspaceId: string, path: string): Promise { + const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/scripts/delete/p/${path}`, { + method: 'POST' + }) + await expectOk(response, `delete script ${path}`) + } + + private async waitForScriptDeployment( + workspaceId: string, + path: string, + hash: string + ): Promise { + const deadline = Date.now() + this.settings.maxWaitMs + + while (Date.now() < deadline) { + const response = await this.request( + `/w/${encodeURIComponent(workspaceId)}/scripts/deployment_status/h/${encodeURIComponent(hash)}` + ) + await expectOk(response, `check deployment status for script ${path}`) + const deployment = (await response.json()) as ScriptDeploymentStatus + if (deployment.lock != null) { + return + } + if (deployment.lock_error_logs) { + throw new Error(`Script deployment failed for ${path}: ${deployment.lock_error_logs}`) + } + await new Promise((resolve) => setTimeout(resolve, this.settings.pollIntervalMs)) + } + + throw new Error(`Timed out waiting for script ${path} (${hash}) to deploy`) + } + + private async request(path: string, init?: RequestInit): Promise { + const token = await this.getToken() + return await fetch(`${this.settings.baseUrl}/api${path}`, { + ...init, + headers: { + Authorization: `Bearer ${token}`, + ...(init?.headers ?? {}) + } + }) + } + + private async getToken(): Promise { + const cacheKey = `${this.settings.baseUrl}|${this.settings.email}` + let tokenPromise = tokenCache.get(cacheKey) + if (!tokenPromise) { + tokenPromise = this.login().catch((error) => { + if (tokenCache.get(cacheKey) === tokenPromise) { + tokenCache.delete(cacheKey) + } + throw error + }) + tokenCache.set(cacheKey, tokenPromise) + } + return await tokenPromise + } + + private async login(): Promise { + const response = await fetch(`${this.settings.baseUrl}/api/auth/login`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + email: this.settings.email, + password: this.settings.password + }) + }) + await expectOk(response, 'login for backend validation') + return (await response.text()).trim() + } +} + +async function withSharedWorkspaceLock(workspaceId: string, body: () => Promise): Promise { + const previous = sharedWorkspaceQueue.get(workspaceId) ?? Promise.resolve() + let releaseCurrent: (() => void) | undefined + const current = new Promise((resolve) => { + releaseCurrent = resolve + }) + const tail = previous.catch(() => undefined).then(() => current) + sharedWorkspaceQueue.set(workspaceId, tail) + + await previous.catch(() => undefined) + + try { + return await body() + } finally { + releaseCurrent?.() + if (sharedWorkspaceQueue.get(workspaceId) === tail) { + sharedWorkspaceQueue.delete(workspaceId) + } + } +} + +function buildWorkspaceId(prefix: string, caseId: string, attempt: number): string { + const caseSlug = caseId + .toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 30) + const suffix = randomUUID().slice(0, 8) + return `${prefix}-${caseSlug || 'case'}-a${attempt}-${suffix}` +} + +function extractFolderName(path: string): string | null { + if (!path.startsWith('f/')) { + return null + } + const segments = path.split('/').slice(1, -1) + return segments.length > 0 ? segments.join('/') : null +} + +function withQuery( + path: string, + params: Record +): string { + const query = new URLSearchParams() + for (const [key, value] of Object.entries(params)) { + if (value === undefined) { + continue + } + query.set(key, String(value)) + } + const suffix = query.toString() + return suffix ? `${path}?${suffix}` : path +} + +async function expectOk(response: Response, context: string): Promise { + if (response.ok) { + return + } + throw new Error(`${context} failed: ${response.status} ${response.statusText} - ${await response.text()}`) +} + +function readStringField( + value: Record, + field: string, + context: string +): string { + const candidate = value[field] + if (typeof candidate === 'string' && candidate.length > 0) { + return candidate + } + throw new Error(`${context} is missing string field ${field}`) +} + +function isConflictMessage(message: string): boolean { + const normalized = message.toLowerCase() + return normalized.includes('already exists') || normalized.includes('path conflict') +} + +function isManagedSharedWorkspacePath(path: string): boolean { + return managedSharedWorkspacePrefixes.some((prefix) => path.startsWith(prefix)) +} diff --git a/ai_evals/adapters/frontend/benchmarkRunner.ts b/ai_evals/adapters/frontend/benchmarkRunner.ts index 33b1555654..6330211f98 100644 --- a/ai_evals/adapters/frontend/benchmarkRunner.ts +++ b/ai_evals/adapters/frontend/benchmarkRunner.ts @@ -1,4 +1,5 @@ import { loadSelectedCases } from "../../core/cases"; +import { resolveBackendValidationSettings } from "../../core/backendValidation"; import { formatRunModelLabel, getFrontendEvalModel, @@ -22,9 +23,13 @@ export async function runFrontendBenchmarkFromEnv(): Promise const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1"; const verbose = process.env.WMILL_FRONTEND_AI_EVAL_VERBOSE === "1"; const model = resolveEvalModel(mode, process.env.WMILL_FRONTEND_AI_EVAL_MODEL); + const backendValidation = resolveBackendValidationSettings({ + evalMode: mode, + requestedMode: process.env.WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION, + }); const selectedCases = await loadSelectedCases(mode, caseIds); - const modeRunner = getModeRunner(mode, getFrontendEvalModel(model)); + const modeRunner = getModeRunner(mode, getFrontendEvalModel(model), backendValidation); const runModel = formatRunModelLabel(mode, model); const caseResults = await runSuite({ modeRunner, @@ -48,15 +53,16 @@ export async function runFrontendBenchmarkFromEnv(): Promise function getModeRunner( mode: FrontendBenchmarkMode, - model: ReturnType + model: ReturnType, + backendValidation: ReturnType ): ModeRunner { switch (mode) { case "flow": - return createFlowModeRunner(model); + return createFlowModeRunner(model, backendValidation); case "app": return createAppModeRunner(model); case "script": - return createScriptModeRunner(model); + return createScriptModeRunner(model, backendValidation); } } diff --git a/ai_evals/adapters/frontend/runtime.ts b/ai_evals/adapters/frontend/runtime.ts index 8828cd63af..228eebede9 100644 --- a/ai_evals/adapters/frontend/runtime.ts +++ b/ai_evals/adapters/frontend/runtime.ts @@ -22,6 +22,7 @@ export async function runFrontendBenchmarkAdapter(input: { runs: number model?: string verbose?: boolean + backendValidation?: string }): Promise { const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-')) const outputPath = path.join(tempDir, 'result.json') @@ -48,7 +49,8 @@ export async function runFrontendBenchmarkAdapter(input: { WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs), WMILL_FRONTEND_AI_EVAL_MODEL: input.model ?? "", WMILL_FRONTEND_AI_EVAL_PROGRESS: '1', - WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0' + WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0', + WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? '' } } ) diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml index cdb53c696a..a5b06805f0 100644 --- a/ai_evals/cases/flow.yaml +++ b/ai_evals/cases/flow.yaml @@ -3,6 +3,11 @@ Create a flow that takes two numbers, `a` and `b`, and returns their sum. Keep it simple and use a single step named `sum_numbers`. expected: ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json + runtime: + backendPreview: + args: + a: 4 + b: 5 judgeChecklist: - "the flow takes `a` and `b` as inputs" - "the main step is named `sum_numbers`" @@ -15,6 +20,11 @@ The flow should take `a` and `b` as inputs and use a single step named `sum_numbers`. initial: ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json expected: ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json + runtime: + backendPreview: + args: + a: 2 + b: 3 judgeChecklist: - "the flow takes `a` and `b` as inputs" - "the main step is named `sum_numbers`" @@ -27,6 +37,11 @@ Use a single step named `call_add_numbers`. initial: ai_evals/fixtures/frontend/flow/initial/test2_call_existing_subflow_initial.json expected: ai_evals/fixtures/frontend/flow/expected/test2_call_existing_subflow.json + runtime: + backendPreview: + args: + a: 7 + b: 8 judgeChecklist: - "the parent flow takes `a` and `b` as inputs" - "the main step is named `call_add_numbers`" diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index d64a547f14..8cd5129360 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -2,6 +2,10 @@ import { Command, InvalidArgumentError } from "commander"; import { loadCases, loadSelectedCases } from "../core/cases"; +import { + BACKEND_VALIDATION_MODES, + parseBackendValidationMode, +} from "../core/backendValidation"; import { EVAL_MODELS, type EvalModelSpec, @@ -43,6 +47,7 @@ async function main() { " bun run cli -- run flow --models haiku,opus,4o", " bun run cli -- run flow flow-test0-sum-two-numbers --verbose", " bun run cli -- run flow --record", + " bun run cli -- run flow --backend-validation preview", " bun run cli -- run flow flow-test5-simple-modification --runs 3", " bun run cli -- run cli bun-hello-script", "", @@ -77,6 +82,10 @@ async function main() { .option("--models ", "comma-separated model aliases to run sequentially") .option("--verbose", "stream assistant output during frontend runs") .option("--record", "append a compact summary line to ai_evals/history/.jsonl") + .option( + "--backend-validation ", + `backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})` + ) .action( async ( mode: EvalMode, @@ -88,6 +97,7 @@ async function main() { models?: string; verbose?: boolean; record?: boolean; + backendValidation?: string; } ) => { await handleRun({ @@ -99,6 +109,7 @@ async function main() { models: options.models, verbose: options.verbose ?? false, record: options.record ?? false, + backendValidation: options.backendValidation, }); } ); @@ -143,6 +154,7 @@ async function handleRun(input: { models?: string; verbose: boolean; record: boolean; + backendValidation?: string; }) { if (input.record && input.caseIds.length > 0) { throw new Error("--record only supports full-suite runs; omit case ids to record history"); @@ -153,9 +165,15 @@ async function handleRun(input: { const selectedCases = await loadSelectedCases(input.mode, input.caseIds); const models = resolveRequestedModels(input.mode, input.model, input.models); + const backendValidation = parseBackendValidationMode( + input.backendValidation ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION + ); if (input.outputPath && models.length > 1) { throw new Error("--output only supports a single model run"); } + if (backendValidation !== "off" && input.mode !== "flow" && input.mode !== "script") { + throw new Error("--backend-validation currently supports only flow and script modes"); + } const summaries: Array<{ label: string; passRate: number; averageDurationMs: number }> = []; @@ -177,6 +195,7 @@ async function handleRun(input: { runs: input.runs, model: model.id, verbose: input.verbose, + backendValidation, }); const resolvedOutputPath = diff --git a/ai_evals/core/backendValidation.test.ts b/ai_evals/core/backendValidation.test.ts new file mode 100644 index 0000000000..db4adeef40 --- /dev/null +++ b/ai_evals/core/backendValidation.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "bun:test"; +import { + parseBackendValidationMode, + resolveBackendValidationSettings, +} from "./backendValidation"; + +describe("parseBackendValidationMode", () => { + it("defaults to off", () => { + expect(parseBackendValidationMode(undefined)).toBe("off"); + expect(parseBackendValidationMode("0")).toBe("off"); + expect(parseBackendValidationMode("false")).toBe("off"); + }); + + it("accepts preview aliases", () => { + expect(parseBackendValidationMode("preview")).toBe("preview"); + expect(parseBackendValidationMode("1")).toBe("preview"); + expect(parseBackendValidationMode("true")).toBe("preview"); + }); + + it("rejects unknown modes", () => { + expect(() => parseBackendValidationMode("maybe")).toThrow( + "Unsupported backend validation mode: maybe" + ); + }); +}); + +describe("resolveBackendValidationSettings", () => { + it("rejects unsupported eval modes", () => { + expect(() => + resolveBackendValidationSettings({ + evalMode: "app", + requestedMode: "preview", + }) + ).toThrow('Backend validation mode "preview" is only supported for flow and script evals'); + }); +}); diff --git a/ai_evals/core/backendValidation.ts b/ai_evals/core/backendValidation.ts new file mode 100644 index 0000000000..464be8533a --- /dev/null +++ b/ai_evals/core/backendValidation.ts @@ -0,0 +1,104 @@ +import type { EvalMode } from "./types"; + +export const BACKEND_VALIDATION_MODES = ["off", "preview"] as const; + +export type BackendValidationMode = (typeof BACKEND_VALIDATION_MODES)[number]; + +export interface BackendValidationSettings { + mode: BackendValidationMode; + baseUrl: string; + email: string; + password: string; + keepWorkspaces: boolean; + workspaceOverride?: string; + workspacePrefix: string; + pollIntervalMs: number; + maxWaitMs: number; +} + +export function parseBackendValidationMode(value?: string | null): BackendValidationMode { + const normalized = value?.trim().toLowerCase(); + + if (!normalized || normalized === "off" || normalized === "false" || normalized === "0") { + return "off"; + } + + if (normalized === "preview" || normalized === "true" || normalized === "1") { + return "preview"; + } + + throw new Error( + `Unsupported backend validation mode: ${value}. Use one of: ${BACKEND_VALIDATION_MODES.join(", ")}` + ); +} + +export function resolveBackendValidationSettings(input: { + evalMode: EvalMode; + requestedMode?: string | null; +}): BackendValidationSettings { + const mode = parseBackendValidationMode( + input.requestedMode ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION + ); + + if (mode !== "off" && input.evalMode !== "flow" && input.evalMode !== "script") { + throw new Error( + `Backend validation mode "${mode}" is only supported for flow and script evals` + ); + } + + return { + mode, + baseUrl: normalizeBaseUrl( + process.env.WMILL_AI_EVAL_BACKEND_URL ?? + process.env.WINDMILL_URL ?? + process.env.WINDMILL_BASE_URL ?? + process.env.REMOTE ?? + "http://127.0.0.1:8000" + ), + email: process.env.WMILL_AI_EVAL_BACKEND_EMAIL ?? "admin@windmill.dev", + password: process.env.WMILL_AI_EVAL_BACKEND_PASSWORD ?? "changeme", + keepWorkspaces: isTruthy(process.env.WMILL_AI_EVAL_KEEP_WORKSPACES), + workspaceOverride: sanitizeOptionalWorkspaceId(process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE), + workspacePrefix: sanitizeWorkspacePrefix( + process.env.WMILL_AI_EVAL_WORKSPACE_PREFIX ?? "ai-evals" + ), + pollIntervalMs: parsePositiveInteger( + process.env.WMILL_AI_EVAL_BACKEND_POLL_INTERVAL_MS, + 2000 + ), + maxWaitMs: parsePositiveInteger(process.env.WMILL_AI_EVAL_BACKEND_MAX_WAIT_MS, 120000), + }; +} + +function normalizeBaseUrl(value: string): string { + return value.replace(/\/+$/, ""); +} + +function sanitizeWorkspacePrefix(value: string): string { + const sanitized = value + .trim() + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, ""); + return sanitized.length > 0 ? sanitized : "ai-evals"; +} + +function sanitizeOptionalWorkspaceId(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function isTruthy(value: string | undefined): boolean { + if (!value) { + return false; + } + return ["1", "true", "yes", "on"].includes(value.trim().toLowerCase()); +} + +function parsePositiveInteger(value: string | undefined, fallback: number): number { + if (!value) { + return fallback; + } + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} diff --git a/ai_evals/core/cases.test.ts b/ai_evals/core/cases.test.ts new file mode 100644 index 0000000000..432155d424 --- /dev/null +++ b/ai_evals/core/cases.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "bun:test"; +import { loadCases } from "./cases"; + +describe("loadCases", () => { + it("loads backend preview runtime config for opt-in flow cases", async () => { + const flowCases = await loadCases("flow"); + const caseEntry = flowCases.find((entry) => entry.id === "flow-test1-reuse-existing-script"); + + expect(caseEntry?.runtime).toEqual({ + backendPreview: { + args: { + a: 2, + b: 3, + }, + }, + }); + }); +}); diff --git a/ai_evals/core/cases.ts b/ai_evals/core/cases.ts index 69f1e8a890..cf866df608 100644 --- a/ai_evals/core/cases.ts +++ b/ai_evals/core/cases.ts @@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { parse } from "yaml"; -import type { EvalCase, EvalMode, FlowValidationSpec } from "./types"; +import type { EvalCase, EvalCaseRuntimeSpec, EvalMode, FlowValidationSpec } from "./types"; const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url)); const CASES_DIR = path.join(REPO_ROOT, "ai_evals", "cases"); @@ -14,6 +14,7 @@ interface RawEvalCase { expected?: string; validate?: FlowValidationSpec; judgeChecklist?: string[]; + runtime?: EvalCaseRuntimeSpec; } export function getRepoRoot(): string { @@ -40,6 +41,7 @@ export async function loadCases(mode: EvalMode): Promise { expectedPath: resolveFixturePath(entry.expected), validate: entry.validate, judgeChecklist: entry.judgeChecklist, + runtime: entry.runtime, })); } diff --git a/ai_evals/core/runSuite.ts b/ai_evals/core/runSuite.ts index 9e155298f0..9112f970ea 100644 --- a/ai_evals/core/runSuite.ts +++ b/ai_evals/core/runSuite.ts @@ -155,6 +155,44 @@ async function runCaseAttempts(input: { run, }), ]; + const artifactFiles = input.modeRunner.buildArtifacts?.(run.actual) ?? []; + + if (run.success && input.modeRunner.backendValidate) { + try { + const backendValidation = await input.modeRunner.backendValidate({ + evalCase: input.evalCase, + prompt: input.evalCase.prompt, + initial, + expected, + actual: run.actual, + run, + context: { + caseId: input.evalCase.id, + caseNumber: input.caseIndex + 1, + totalCases: input.totalCases, + attempt, + runs: input.runs, + verbose: input.verbose, + onAssistantMessageStart: undefined, + onAssistantChunk: undefined, + onAssistantMessageEnd: undefined, + }, + }); + + if (backendValidation) { + checks.push(...backendValidation.checks); + artifactFiles.push(...(backendValidation.artifactFiles ?? [])); + } + } catch (error) { + checks.push( + buildCheck( + "backend validation succeeded", + false, + error instanceof Error ? error.message : String(error) + ) + ); + } + } let judgeScore: number | null = null; let judgeSummary: string | null = null; @@ -182,7 +220,6 @@ async function runCaseAttempts(input: { ); } - const artifactFiles = input.modeRunner.buildArtifacts?.(run.actual) ?? []; const attemptResult: BenchmarkAttemptResult = { attempt, passed: checks.every((check) => check.passed), diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index a8ed0baa28..0597dc9c77 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -2,6 +2,15 @@ export const EVAL_MODES = ["cli", "flow", "script", "app"] as const; export type EvalMode = (typeof EVAL_MODES)[number]; +export interface EvalCaseRuntimeBackendPreview { + args?: Record; + timeoutSeconds?: number; +} + +export interface EvalCaseRuntimeSpec { + backendPreview?: EvalCaseRuntimeBackendPreview; +} + export interface FlowValidationSpec { schemaRequiredPaths?: string[]; schemaAnyOf?: Array<{ @@ -23,6 +32,7 @@ export interface EvalCase { expectedPath?: string; validate?: FlowValidationSpec; judgeChecklist?: string[]; + runtime?: EvalCaseRuntimeSpec; } export interface BenchmarkCheck { @@ -43,6 +53,11 @@ export interface BenchmarkArtifactFile { content: string; } +export interface BackendValidationResult { + checks: BenchmarkCheck[]; + artifactFiles?: BenchmarkArtifactFile[]; +} + export interface BenchmarkTokenUsage { prompt: number; completion: number; @@ -91,6 +106,15 @@ export interface ModeRunner { actual: TActual; run: ModeRunOutput; }): BenchmarkCheck[]; + backendValidate?(input: { + evalCase: EvalCase; + prompt: string; + initial: TInitial | undefined; + expected: TExpected | undefined; + actual: TActual; + run: ModeRunOutput; + context: ModeRunContext; + }): Promise; buildArtifacts?(actual: TActual): BenchmarkArtifactFile[]; } diff --git a/ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json b/ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json index f5ab58c476..709e9e93a9 100644 --- a/ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json +++ b/ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json @@ -5,7 +5,7 @@ "id": "sum_numbers", "value": { "type": "script", - "path": "f/evals/add_two_numbers.ts", + "path": "f/evals/add_two_numbers", "input_transforms": { "a": { "type": "javascript", diff --git a/ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json b/ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json index 6540a36c26..9c12a3ff9a 100644 --- a/ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json +++ b/ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json @@ -2,7 +2,7 @@ "workspace": { "scripts": [ { - "path": "f/evals/add_two_numbers.ts", + "path": "f/evals/add_two_numbers", "summary": "Add two numbers", "description": "Returns the sum of two numeric inputs.", "language": "bun", diff --git a/ai_evals/modes/flow.ts b/ai_evals/modes/flow.ts index 36dee80658..4c741bd6db 100644 --- a/ai_evals/modes/flow.ts +++ b/ai_evals/modes/flow.ts @@ -1,4 +1,5 @@ import { readJsonFile } from "../core/files"; +import type { BackendValidationSettings } from "../core/backendValidation"; import type { FrontendEvalModelConfig } from "../core/models"; import { validateFlowState, type FlowState } from "../core/validators"; import type { BenchmarkArtifactFile, ModeRunner } from "../core/types"; @@ -7,6 +8,7 @@ import { type FlowFixture, } from "../adapters/frontend/core/flow/flowEvalRunner"; import type { FlowWorkspaceFixtures } from "../adapters/frontend/core/flow/fileHelpers"; +import { BackendPreviewClient } from "../adapters/frontend/backendPreview"; import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon"; interface FlowInitialFixture { @@ -15,7 +17,8 @@ interface FlowInitialFixture { } export function createFlowModeRunner( - modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL + modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL, + backendValidation?: BackendValidationSettings ): ModeRunner { return { mode: "flow", @@ -61,6 +64,65 @@ export function createFlowModeRunner( validate: evalCase.validate, }); }, + async backendValidate({ evalCase, initial, actual, context }) { + if (backendValidation?.mode !== "preview" || !evalCase.runtime?.backendPreview) { + return null; + } + + if (!actual.value) { + return { + checks: [ + { + name: "backend flow preview succeeded", + passed: false, + details: "Generated flow is missing value.modules", + }, + ], + }; + } + + const previewClient = new BackendPreviewClient(backendValidation); + return await previewClient.withWorkspace(evalCase.id, context.attempt, async (workspaceId) => { + await seedWorkspaceFixtures(previewClient, workspaceId, initial?.workspace); + + const completedJob = await previewClient.runFlowPreview({ + workspaceId, + value: actual.value as Record, + args: evalCase.runtime?.backendPreview?.args ?? {}, + timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds, + }); + + return { + checks: [ + { + name: "backend flow preview succeeded", + passed: completedJob.success, + details: completedJob.success + ? `workspace=${workspaceId}` + : `workspace=${workspaceId}; job=${completedJob.id}`, + }, + ], + artifactFiles: [ + { + path: "backend-preview.json", + content: + JSON.stringify( + { + workspaceId, + jobId: completedJob.id, + success: completedJob.success, + result: completedJob.result, + logs: completedJob.logs, + completedJob: completedJob.raw, + }, + null, + 2 + ) + "\n", + }, + ], + }; + }); + }, buildArtifacts(actual): BenchmarkArtifactFile[] { return [ { @@ -102,3 +164,32 @@ function normalizeFlowStateFixture(value: unknown): FlowState { function isObject(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } + +async function seedWorkspaceFixtures( + previewClient: BackendPreviewClient, + workspaceId: string, + fixtures?: FlowWorkspaceFixtures +): Promise { + for (const script of fixtures?.scripts ?? []) { + await previewClient.createScript({ + workspaceId, + path: script.path, + summary: script.summary, + description: script.description, + schema: script.schema, + content: script.content, + language: script.language, + }); + } + + for (const flow of fixtures?.flows ?? []) { + await previewClient.createFlow({ + workspaceId, + path: flow.path, + summary: flow.summary, + description: flow.description, + schema: flow.schema, + value: flow.value as Record, + }); + } +} diff --git a/ai_evals/modes/script.ts b/ai_evals/modes/script.ts index f3ab232cc3..1670488734 100644 --- a/ai_evals/modes/script.ts +++ b/ai_evals/modes/script.ts @@ -1,13 +1,16 @@ import { readJsonFile } from "../core/files"; +import type { BackendValidationSettings } from "../core/backendValidation"; import type { FrontendEvalModelConfig } from "../core/models"; import { validateScriptState } from "../core/validators"; import type { BenchmarkArtifactFile, ModeRunner } from "../core/types"; +import { BackendPreviewClient } from "../adapters/frontend/backendPreview"; import { runScriptEval } from "../adapters/frontend/core/script/scriptEvalRunner"; import type { ScriptEvalState } from "../adapters/frontend/core/script/fileHelpers"; import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon"; export function createScriptModeRunner( - modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL + modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL, + backendValidation?: BackendValidationSettings ): ModeRunner { return { mode: "script", @@ -45,6 +48,57 @@ export function createScriptModeRunner( validate({ actual, initial, expected }) { return validateScriptState({ actual, initial, expected }); }, + async backendValidate({ evalCase, initial, actual, context }) { + if (backendValidation?.mode !== "preview") { + return null; + } + + const previewClient = new BackendPreviewClient(backendValidation); + return await previewClient.withWorkspace(evalCase.id, context.attempt, async (workspaceId) => { + const completedJob = await previewClient.runScriptPreview({ + workspaceId, + content: actual.code, + args: + (evalCase.runtime?.backendPreview?.args as Record | undefined) ?? + actual.args ?? + initial?.args ?? + {}, + language: normalizePreviewLanguage(actual.lang), + path: toPreviewScriptPath(actual.path), + timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds, + }); + + return { + checks: [ + { + name: "backend script preview succeeded", + passed: completedJob.success, + details: completedJob.success + ? `workspace=${workspaceId}` + : `workspace=${workspaceId}; job=${completedJob.id}`, + }, + ], + artifactFiles: [ + { + path: "backend-preview.json", + content: + JSON.stringify( + { + workspaceId, + jobId: completedJob.id, + success: completedJob.success, + result: completedJob.result, + logs: completedJob.logs, + completedJob: completedJob.raw, + }, + null, + 2 + ) + "\n", + }, + ], + }; + }); + }, buildArtifacts(actual): BenchmarkArtifactFile[] { return [ { @@ -59,3 +113,11 @@ export function createScriptModeRunner( }, }; } + +function normalizePreviewLanguage(language: ScriptEvalState["lang"]): string { + return language === "bunnative" ? "bun" : language; +} + +function toPreviewScriptPath(filePath: string): string { + return filePath.replace(/\.[^.\/]+$/, ""); +}