chore: add backend preview validation to ai evals (#8827)

* feat: add backend preview validation to ai evals

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: refresh shared preview workspace assets

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: harden shared backend preview validation

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

ai_evals/README.md

@@ -55,6 +55,7 @@ bun run cli -- run flow flow-test4-order-processing-loop --model opus
 bun run cli -- run flow flow-test0-sum-two-numbers --models haiku,opus,4o
 bun run cli -- run flow flow-test0-sum-two-numbers --runs 3 --verbose
 bun run cli -- run flow --record
+WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000 bun run cli -- run flow --backend-validation preview
 bun run cli -- run cli bun-hello-script
 ```
 
@@ -72,6 +73,7 @@ Public CLI surface:
 - `--models <a,b,c>`: run the same cases sequentially against several model aliases
 - `--verbose`: stream assistant output for frontend runs
 - `--record`: append a compact tracked summary line to `ai_evals/history/<mode>.jsonl` for full-suite runs only
+- `--backend-validation <mode>`: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals
 
 ## Models
 
@@ -114,6 +116,7 @@ Optional fields:
 - `initial`: starting state fixture
 - `expected`: expected artifact fixture
 - `validate`: extra deterministic validation rules
+- `runtime.backendPreview`: optional real backend preview config for smoke validation
 
 For `flow` mode, `validate` can express requirements such as:
 
@@ -125,6 +128,23 @@ For `flow` mode, an `initial` fixture can also include a benchmark workspace cat
 existing scripts and flows. That lets the real `search_workspace` and
 `get_runnable_details` tools discover reusable workspace runnables during evals.
 
+If `--backend-validation preview` is enabled:
+
+- `script` evals run a real backend script preview in an isolated temp workspace
+- `flow` evals run a real backend flow preview only for cases that define `runtime.backendPreview`
+- `flow` cases with `initial.workspace` fixtures seed those scripts and flows into the preview workspace before preview
+- when `WMILL_AI_EVAL_BACKEND_WORKSPACE` is set, `ai_evals` treats that workspace as a dedicated test workspace, clears managed eval assets under `f/evals/*` before each preview run, and then reseeds the current case fixtures
+
+Supported backend validation env vars:
+
+- `WMILL_AI_EVAL_BACKEND_VALIDATION=preview`
+- `WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000`
+- `WMILL_AI_EVAL_BACKEND_EMAIL=admin@windmill.dev`
+- `WMILL_AI_EVAL_BACKEND_PASSWORD=changeme`
+- `WMILL_AI_EVAL_BACKEND_WORKSPACE=integration-tests` to reuse an existing workspace on CE installs with low workspace limits
+- `WMILL_AI_EVAL_KEEP_WORKSPACES=1`
+- `WMILL_AI_EVAL_WORKSPACE_PREFIX=ai-evals`
+
 ## Results And Artifacts
 
 Every run writes:
@@ -158,6 +178,7 @@ Typical artifacts by mode:
 - `script`: `script.json` plus the generated script file
 - `app`: `app.json` plus frontend/backend files
 - `cli`: `assistant-output.txt` plus generated workspace files
+- backend-validated attempts also include `backend-preview.json`
 
 ## Layout
 

ai_evals/adapters/frontend/backendPreview.test.ts

@@ -0,0 +1,246 @@
+import { afterEach, describe, expect, it } from 'bun:test'
+import type { BackendValidationSettings } from '../../core/backendValidation'
+import { BackendPreviewClient } from './backendPreview'
+
+const ORIGINAL_FETCH = globalThis.fetch
+
+afterEach(() => {
+	globalThis.fetch = ORIGINAL_FETCH
+})
+
+describe('BackendPreviewClient', () => {
+	it('updates an existing seeded script on path conflict and waits for deployment', async () => {
+		const requests: Array<{ url: string; init?: RequestInit }> = []
+		globalThis.fetch = mockFetch(
+			requests,
+			textResponse(200, 'token'),
+			textResponse(200, ''),
+			textResponse(400, 'Path conflict for f/evals/add_two_numbers with non-archived hash 123'),
+			jsonResponse(200, { hash: '123' }),
+			textResponse(200, '456'),
+			jsonResponse(200, { lock: 'script.lock', lock_error_logs: null })
+		)
+
+		const client = new BackendPreviewClient(
+			buildSettings({ baseUrl: 'http://backend.test/script-upsert' })
+		)
+
+		await client.createScript({
+			workspaceId: 'test',
+			path: 'f/evals/add_two_numbers',
+			summary: 'Add two numbers',
+			content: 'export async function main(a: number, b: number) { return a + b }',
+			language: 'bun'
+		})
+
+		expect(requests.map((entry) => entry.url)).toEqual([
+			'http://backend.test/script-upsert/api/auth/login',
+			'http://backend.test/script-upsert/api/w/test/folders/create',
+			'http://backend.test/script-upsert/api/w/test/scripts/create',
+			'http://backend.test/script-upsert/api/w/test/scripts/get/p/f/evals/add_two_numbers',
+			'http://backend.test/script-upsert/api/w/test/scripts/create',
+			'http://backend.test/script-upsert/api/w/test/scripts/deployment_status/h/456'
+		])
+
+		const updateRequest = requests[4]
+		expect(updateRequest.init?.method).toBe('POST')
+		expect(JSON.parse(String(updateRequest.init?.body))).toMatchObject({
+			path: 'f/evals/add_two_numbers',
+			parent_hash: '123',
+			language: 'bun'
+		})
+	})
+
+	it('updates an existing seeded flow on create conflict', async () => {
+		const requests: Array<{ url: string; init?: RequestInit }> = []
+		globalThis.fetch = mockFetch(
+			requests,
+			textResponse(200, 'token'),
+			textResponse(200, ''),
+			textResponse(400, 'Flow f/evals/add_numbers_flow already exists'),
+			textResponse(200, '')
+		)
+
+		const client = new BackendPreviewClient(
+			buildSettings({ baseUrl: 'http://backend.test/flow-upsert' })
+		)
+
+		await client.createFlow({
+			workspaceId: 'test',
+			path: 'f/evals/add_numbers_flow',
+			summary: 'Add numbers',
+			value: { modules: [] }
+		})
+
+		expect(requests.map((entry) => entry.url)).toEqual([
+			'http://backend.test/flow-upsert/api/auth/login',
+			'http://backend.test/flow-upsert/api/w/test/folders/create',
+			'http://backend.test/flow-upsert/api/w/test/flows/create',
+			'http://backend.test/flow-upsert/api/w/test/flows/update/f/evals/add_numbers_flow'
+		])
+
+		const updateRequest = requests[3]
+		expect(updateRequest.init?.method).toBe('POST')
+		expect(JSON.parse(String(updateRequest.init?.body))).toMatchObject({
+			path: 'f/evals/add_numbers_flow',
+			value: { modules: [] }
+		})
+	})
+
+	it('serializes shared-workspace validations inside the overridden workspace', async () => {
+		globalThis.fetch = async (input) => {
+			const url = String(input)
+			if (url.endsWith('/api/auth/login')) {
+				return textResponse(200, 'token')
+			}
+			if (url.endsWith('/api/workspaces/exists')) {
+				return textResponse(200, 'true')
+			}
+			if (url.endsWith('/api/w/shared-preview/flows/list_paths')) {
+				return jsonResponse(200, [])
+			}
+			if (url.endsWith('/api/w/shared-preview/scripts/list_paths')) {
+				return jsonResponse(200, [])
+			}
+			throw new Error(`Unexpected fetch: ${url}`)
+		}
+
+		const client = new BackendPreviewClient(
+			buildSettings({
+				baseUrl: 'http://backend.test/shared-lock',
+				workspaceOverride: 'shared-preview'
+			})
+		)
+
+		const order: string[] = []
+		let releaseFirst: (() => void) | undefined
+		let notifyFirstStart: (() => void) | undefined
+		const firstStarted = new Promise<void>((resolve) => {
+			notifyFirstStart = resolve
+		})
+
+		const first = client.withWorkspace('flow-test1', 1, async () => {
+			order.push('first:start')
+			notifyFirstStart?.()
+			await new Promise<void>((resolve) => {
+				releaseFirst = resolve
+			})
+			order.push('first:end')
+		})
+
+		const second = client.withWorkspace('flow-test2', 1, async () => {
+			order.push('second:start')
+			order.push('second:end')
+		})
+
+		await firstStarted
+		expect(order).toEqual(['first:start'])
+
+		releaseFirst?.()
+		await Promise.all([first, second])
+
+		expect(order).toEqual(['first:start', 'first:end', 'second:start', 'second:end'])
+	})
+
+	it('clears managed shared-workspace assets before preview runs', async () => {
+		const requests: Array<{ url: string; init?: RequestInit }> = []
+		globalThis.fetch = mockFetch(
+			requests,
+			textResponse(200, 'token'),
+			textResponse(200, 'true'),
+			jsonResponse(200, ['f/evals/old_subflow', 'u/admin/keep_flow']),
+			textResponse(200, ''),
+			jsonResponse(200, ['f/evals/old_script', 'f/shared/keep_script']),
+			textResponse(200, '')
+		)
+
+		const client = new BackendPreviewClient(
+			buildSettings({
+				baseUrl: 'http://backend.test/shared-cleanup',
+				workspaceOverride: 'shared-preview'
+			})
+		)
+
+		await client.withWorkspace('flow-test1', 1, async () => undefined)
+
+		expect(requests.map((entry) => entry.url)).toEqual([
+			'http://backend.test/shared-cleanup/api/auth/login',
+			'http://backend.test/shared-cleanup/api/workspaces/exists',
+			'http://backend.test/shared-cleanup/api/w/shared-preview/flows/list_paths',
+			'http://backend.test/shared-cleanup/api/w/shared-preview/flows/delete/f/evals/old_subflow',
+			'http://backend.test/shared-cleanup/api/w/shared-preview/scripts/list_paths',
+			'http://backend.test/shared-cleanup/api/w/shared-preview/scripts/delete/p/f/evals/old_script'
+		])
+	})
+
+	it('retries login after a cached login failure', async () => {
+		const requests: Array<{ url: string; init?: RequestInit }> = []
+		globalThis.fetch = mockFetch(
+			requests,
+			textResponse(503, 'backend starting'),
+			textResponse(200, 'token'),
+			textResponse(200, 'true'),
+			jsonResponse(200, []),
+			jsonResponse(200, [])
+		)
+
+		const client = new BackendPreviewClient(
+			buildSettings({
+				baseUrl: 'http://backend.test/login-retry',
+				workspaceOverride: 'shared-preview'
+			})
+		)
+
+		await expect(client.withWorkspace('flow-test1', 1, async () => undefined)).rejects.toThrow(
+			'login for backend validation failed'
+		)
+		await expect(client.withWorkspace('flow-test1', 1, async () => 'ok')).resolves.toBe('ok')
+
+		expect(
+			requests.filter((entry) => entry.url === 'http://backend.test/login-retry/api/auth/login')
+		).toHaveLength(2)
+	})
+})
+
+function buildSettings(
+	overrides: Partial<BackendValidationSettings> = {}
+): BackendValidationSettings {
+	return {
+		mode: 'preview',
+		baseUrl: 'http://backend.test/default',
+		email: 'admin@windmill.dev',
+		password: 'changeme',
+		keepWorkspaces: true,
+		workspacePrefix: 'ai-evals',
+		pollIntervalMs: 1,
+		maxWaitMs: 50,
+		...overrides
+	}
+}
+
+function mockFetch(
+	requests: Array<{ url: string; init?: RequestInit }>,
+	...responses: Response[]
+): typeof fetch {
+	const queue = [...responses]
+	return async (input, init) => {
+		const url = String(input)
+		requests.push({ url, init })
+		const next = queue.shift()
+		if (!next) {
+			throw new Error(`Unexpected fetch: ${url}`)
+		}
+		return next
+	}
+}
+
+function jsonResponse(status: number, body: unknown): Response {
+	return new Response(JSON.stringify(body), {
+		status,
+		headers: { 'Content-Type': 'application/json' }
+	})
+}
+
+function textResponse(status: number, body: string): Response {
+	return new Response(body, { status })
+}

ai_evals/adapters/frontend/backendPreview.ts

@@ -0,0 +1,502 @@
+import { randomUUID } from 'node:crypto'
+import type { BackendValidationSettings } from '../../core/backendValidation'
+
+interface CompletedJobResultMaybe {
+	completed: boolean
+	result: unknown
+	success?: boolean
+	started?: boolean
+}
+
+interface ScriptDeploymentStatus {
+	lock?: unknown
+	lock_error_logs?: string | null
+}
+
+export interface CompletedPreviewJob {
+	id: string
+	success: boolean
+	result: unknown
+	logs?: string | null
+	raw: Record<string, unknown>
+}
+
+const tokenCache = new Map<string, Promise<string>>()
+const sharedWorkspaceQueue = new Map<string, Promise<void>>()
+const managedSharedWorkspacePrefixes = ['f/evals/']
+
+export class BackendPreviewClient {
+	constructor(private readonly settings: BackendValidationSettings) {}
+
+	async withWorkspace<T>(
+		caseId: string,
+		attempt: number,
+		body: (workspaceId: string) => Promise<T>
+	): Promise<T> {
+		const workspaceId =
+			this.settings.workspaceOverride ??
+			buildWorkspaceId(this.settings.workspacePrefix, caseId, attempt)
+
+		const run = async () => {
+			await this.ensureWorkspace(workspaceId)
+			if (this.settings.workspaceOverride) {
+				await this.clearManagedSharedWorkspaceAssets(workspaceId)
+			}
+
+			try {
+				return await body(workspaceId)
+			} finally {
+				if (!this.settings.keepWorkspaces && !this.settings.workspaceOverride) {
+					await this.deleteWorkspace(workspaceId).catch(() => undefined)
+				}
+			}
+		}
+
+		if (this.settings.workspaceOverride) {
+			return await withSharedWorkspaceLock(workspaceId, run)
+		}
+
+		return await run()
+	}
+
+	async createScript(input: {
+		workspaceId: string
+		path: string
+		summary: string
+		description?: string
+		schema?: Record<string, unknown>
+		content: string
+		language: string
+	}): Promise<void> {
+		await this.ensureFolderForPath(input.workspaceId, input.path)
+
+		const payload = {
+			path: input.path,
+			summary: input.summary,
+			description: input.description ?? '',
+			content: input.content,
+			schema: input.schema ?? { type: 'object', properties: {}, required: [] },
+			is_template: false,
+			language: input.language,
+			kind: 'script'
+		}
+
+		const response = await this.request(`/w/${encodeURIComponent(input.workspaceId)}/scripts/create`, {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/json' },
+			body: JSON.stringify(payload)
+		})
+
+		if (response.ok) {
+			await this.waitForScriptDeployment(input.workspaceId, input.path, (await response.text()).trim())
+			return
+		}
+
+		const message = await response.text()
+		if (!isConflictMessage(message)) {
+			throw new Error(`create script ${input.path} failed: ${response.status} ${response.statusText} - ${message}`)
+		}
+
+		const currentScript = await this.getScriptByPath(input.workspaceId, input.path)
+		const currentHash = readStringField(currentScript, 'hash', `script ${input.path}`)
+		const updateResponse = await this.request(
+			`/w/${encodeURIComponent(input.workspaceId)}/scripts/create`,
+			{
+				method: 'POST',
+				headers: { 'Content-Type': 'application/json' },
+				body: JSON.stringify({
+					...payload,
+					parent_hash: currentHash
+				})
+			}
+		)
+		await expectOk(updateResponse, `update script ${input.path}`)
+		await this.waitForScriptDeployment(input.workspaceId, input.path, (await updateResponse.text()).trim())
+	}
+
+	async createFlow(input: {
+		workspaceId: string
+		path: string
+		summary: string
+		description?: string
+		schema?: Record<string, unknown>
+		value: Record<string, unknown>
+	}): Promise<void> {
+		await this.ensureFolderForPath(input.workspaceId, input.path)
+
+		const payload = {
+			path: input.path,
+			summary: input.summary,
+			description: input.description ?? '',
+			schema: input.schema ?? { type: 'object', properties: {}, required: [] },
+			value: input.value
+		}
+
+		const response = await this.request(`/w/${encodeURIComponent(input.workspaceId)}/flows/create`, {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/json' },
+			body: JSON.stringify(payload)
+		})
+
+		if (response.ok) {
+			return
+		}
+
+		const message = await response.text()
+		if (!isConflictMessage(message)) {
+			throw new Error(`create flow ${input.path} failed: ${response.status} ${response.statusText} - ${message}`)
+		}
+
+		const updateResponse = await this.request(
+			`/w/${encodeURIComponent(input.workspaceId)}/flows/update/${input.path}`,
+			{
+				method: 'POST',
+				headers: { 'Content-Type': 'application/json' },
+				body: JSON.stringify(payload)
+			}
+		)
+		await expectOk(updateResponse, `update flow ${input.path}`)
+	}
+
+	async runScriptPreview(input: {
+		workspaceId: string
+		content: string
+		args: Record<string, unknown>
+		language: string
+		path?: string
+		timeoutSeconds?: number
+	}): Promise<CompletedPreviewJob> {
+		const response = await this.request(
+			withQuery(`/w/${encodeURIComponent(input.workspaceId)}/jobs/run/preview`, {
+				timeout: input.timeoutSeconds
+			}),
+			{
+				method: 'POST',
+				headers: { 'Content-Type': 'application/json' },
+				body: JSON.stringify({
+					content: input.content,
+					args: input.args,
+					language: input.language,
+					path: input.path
+				})
+			}
+		)
+
+		await expectOk(response, 'start script preview')
+		const jobId = (await response.text()).trim()
+		return await this.waitForCompletedJob(input.workspaceId, jobId)
+	}
+
+	async runFlowPreview(input: {
+		workspaceId: string
+		value: Record<string, unknown>
+		args: Record<string, unknown>
+		timeoutSeconds?: number
+		path?: string
+	}): Promise<CompletedPreviewJob> {
+		const response = await this.request(
+			withQuery(`/w/${encodeURIComponent(input.workspaceId)}/jobs/run/preview_flow`, {
+				timeout: input.timeoutSeconds
+			}),
+			{
+				method: 'POST',
+				headers: { 'Content-Type': 'application/json' },
+				body: JSON.stringify({
+					value: input.value,
+					args: input.args,
+					path: input.path
+				})
+			}
+		)
+
+		await expectOk(response, 'start flow preview')
+		const jobId = (await response.text()).trim()
+		return await this.waitForCompletedJob(input.workspaceId, jobId)
+	}
+
+	private async ensureWorkspace(workspaceId: string): Promise<void> {
+		const existsResponse = await this.request('/workspaces/exists', {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/json' },
+			body: JSON.stringify({ id: workspaceId })
+		})
+		await expectOk(existsResponse, `check workspace ${workspaceId}`)
+
+		if ((await existsResponse.text()).trim() === 'true') {
+			return
+		}
+
+		const createResponse = await this.request('/workspaces/create', {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/json' },
+			body: JSON.stringify({ id: workspaceId, name: workspaceId })
+		})
+		try {
+			await expectOk(createResponse, `create workspace ${workspaceId}`)
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error)
+			if (message.includes('maximum number of workspaces')) {
+				throw new Error(
+					`${message}. Reuse an existing workspace with WMILL_AI_EVAL_BACKEND_WORKSPACE=<workspace-id>.`
+				)
+			}
+			throw error
+		}
+	}
+
+	private async deleteWorkspace(workspaceId: string): Promise<void> {
+		const response = await this.request(`/workspaces/delete/${encodeURIComponent(workspaceId)}`, {
+			method: 'DELETE'
+		})
+		await expectOk(response, `delete workspace ${workspaceId}`)
+	}
+
+	private async ensureFolderForPath(workspaceId: string, path: string): Promise<void> {
+		const folderName = extractFolderName(path)
+		if (!folderName) {
+			return
+		}
+
+		const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/folders/create`, {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/json' },
+			body: JSON.stringify({ name: folderName })
+		})
+
+		if (response.ok) {
+			return
+		}
+
+		const message = await response.text()
+		if (!message.toLowerCase().includes('already exists')) {
+			throw new Error(`Failed to create folder ${folderName}: ${message}`)
+		}
+	}
+
+	private async waitForCompletedJob(
+		workspaceId: string,
+		jobId: string
+	): Promise<CompletedPreviewJob> {
+		const deadline = Date.now() + this.settings.maxWaitMs
+
+		while (Date.now() < deadline) {
+			const maybeResponse = await this.request(
+				`/w/${encodeURIComponent(workspaceId)}/jobs_u/completed/get_result_maybe/${encodeURIComponent(jobId)}?get_started=false`
+			)
+			await expectOk(maybeResponse, `poll job ${jobId}`)
+			const maybeResult = (await maybeResponse.json()) as CompletedJobResultMaybe
+
+			if (maybeResult.completed) {
+				const completedResponse = await this.request(
+					`/w/${encodeURIComponent(workspaceId)}/jobs_u/completed/get/${encodeURIComponent(jobId)}`
+				)
+				await expectOk(completedResponse, `get completed job ${jobId}`)
+				const completedJob = (await completedResponse.json()) as Record<string, unknown>
+				return {
+					id: jobId,
+					success: Boolean(maybeResult.success),
+					result: maybeResult.result,
+					logs:
+						typeof completedJob.logs === 'string' || completedJob.logs === null
+							? (completedJob.logs as string | null)
+							: null,
+					raw: completedJob
+				}
+			}
+
+			await new Promise((resolve) => setTimeout(resolve, this.settings.pollIntervalMs))
+		}
+
+		throw new Error(`Timed out waiting for preview job ${jobId} to complete`)
+	}
+
+	private async getScriptByPath(workspaceId: string, path: string): Promise<Record<string, unknown>> {
+		const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/scripts/get/p/${path}`)
+		await expectOk(response, `get script ${path}`)
+		return (await response.json()) as Record<string, unknown>
+	}
+
+	private async clearManagedSharedWorkspaceAssets(workspaceId: string): Promise<void> {
+		const flowPaths = await this.listFlowPaths(workspaceId)
+		for (const path of flowPaths.filter(isManagedSharedWorkspacePath)) {
+			await this.deleteFlowByPath(workspaceId, path)
+		}
+
+		const scriptPaths = await this.listScriptPaths(workspaceId)
+		for (const path of scriptPaths.filter(isManagedSharedWorkspacePath)) {
+			await this.deleteScriptByPath(workspaceId, path)
+		}
+	}
+
+	private async listFlowPaths(workspaceId: string): Promise<string[]> {
+		const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/flows/list_paths`)
+		await expectOk(response, `list flows in workspace ${workspaceId}`)
+		return await response.json()
+	}
+
+	private async listScriptPaths(workspaceId: string): Promise<string[]> {
+		const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/scripts/list_paths`)
+		await expectOk(response, `list scripts in workspace ${workspaceId}`)
+		return await response.json()
+	}
+
+	private async deleteFlowByPath(workspaceId: string, path: string): Promise<void> {
+		const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/flows/delete/${path}`, {
+			method: 'DELETE'
+		})
+		await expectOk(response, `delete flow ${path}`)
+	}
+
+	private async deleteScriptByPath(workspaceId: string, path: string): Promise<void> {
+		const response = await this.request(`/w/${encodeURIComponent(workspaceId)}/scripts/delete/p/${path}`, {
+			method: 'POST'
+		})
+		await expectOk(response, `delete script ${path}`)
+	}
+
+	private async waitForScriptDeployment(
+		workspaceId: string,
+		path: string,
+		hash: string
+	): Promise<void> {
+		const deadline = Date.now() + this.settings.maxWaitMs
+
+		while (Date.now() < deadline) {
+			const response = await this.request(
+				`/w/${encodeURIComponent(workspaceId)}/scripts/deployment_status/h/${encodeURIComponent(hash)}`
+			)
+			await expectOk(response, `check deployment status for script ${path}`)
+			const deployment = (await response.json()) as ScriptDeploymentStatus
+			if (deployment.lock != null) {
+				return
+			}
+			if (deployment.lock_error_logs) {
+				throw new Error(`Script deployment failed for ${path}: ${deployment.lock_error_logs}`)
+			}
+			await new Promise((resolve) => setTimeout(resolve, this.settings.pollIntervalMs))
+		}
+
+		throw new Error(`Timed out waiting for script ${path} (${hash}) to deploy`)
+	}
+
+	private async request(path: string, init?: RequestInit): Promise<Response> {
+		const token = await this.getToken()
+		return await fetch(`${this.settings.baseUrl}/api${path}`, {
+			...init,
+			headers: {
+				Authorization: `Bearer ${token}`,
+				...(init?.headers ?? {})
+			}
+		})
+	}
+
+	private async getToken(): Promise<string> {
+		const cacheKey = `${this.settings.baseUrl}|${this.settings.email}`
+		let tokenPromise = tokenCache.get(cacheKey)
+		if (!tokenPromise) {
+			tokenPromise = this.login().catch((error) => {
+				if (tokenCache.get(cacheKey) === tokenPromise) {
+					tokenCache.delete(cacheKey)
+				}
+				throw error
+			})
+			tokenCache.set(cacheKey, tokenPromise)
+		}
+		return await tokenPromise
+	}
+
+	private async login(): Promise<string> {
+		const response = await fetch(`${this.settings.baseUrl}/api/auth/login`, {
+			method: 'POST',
+			headers: { 'Content-Type': 'application/json' },
+			body: JSON.stringify({
+				email: this.settings.email,
+				password: this.settings.password
+			})
+		})
+		await expectOk(response, 'login for backend validation')
+		return (await response.text()).trim()
+	}
+}
+
+async function withSharedWorkspaceLock<T>(workspaceId: string, body: () => Promise<T>): Promise<T> {
+	const previous = sharedWorkspaceQueue.get(workspaceId) ?? Promise.resolve()
+	let releaseCurrent: (() => void) | undefined
+	const current = new Promise<void>((resolve) => {
+		releaseCurrent = resolve
+	})
+	const tail = previous.catch(() => undefined).then(() => current)
+	sharedWorkspaceQueue.set(workspaceId, tail)
+
+	await previous.catch(() => undefined)
+
+	try {
+		return await body()
+	} finally {
+		releaseCurrent?.()
+		if (sharedWorkspaceQueue.get(workspaceId) === tail) {
+			sharedWorkspaceQueue.delete(workspaceId)
+		}
+	}
+}
+
+function buildWorkspaceId(prefix: string, caseId: string, attempt: number): string {
+	const caseSlug = caseId
+		.toLowerCase()
+		.replace(/[^a-z0-9-]+/g, '-')
+		.replace(/^-+|-+$/g, '')
+		.slice(0, 30)
+	const suffix = randomUUID().slice(0, 8)
+	return `${prefix}-${caseSlug || 'case'}-a${attempt}-${suffix}`
+}
+
+function extractFolderName(path: string): string | null {
+	if (!path.startsWith('f/')) {
+		return null
+	}
+	const segments = path.split('/').slice(1, -1)
+	return segments.length > 0 ? segments.join('/') : null
+}
+
+function withQuery(
+	path: string,
+	params: Record<string, string | number | undefined>
+): string {
+	const query = new URLSearchParams()
+	for (const [key, value] of Object.entries(params)) {
+		if (value === undefined) {
+			continue
+		}
+		query.set(key, String(value))
+	}
+	const suffix = query.toString()
+	return suffix ? `${path}?${suffix}` : path
+}
+
+async function expectOk(response: Response, context: string): Promise<void> {
+	if (response.ok) {
+		return
+	}
+	throw new Error(`${context} failed: ${response.status} ${response.statusText} - ${await response.text()}`)
+}
+
+function readStringField(
+	value: Record<string, unknown>,
+	field: string,
+	context: string
+): string {
+	const candidate = value[field]
+	if (typeof candidate === 'string' && candidate.length > 0) {
+		return candidate
+	}
+	throw new Error(`${context} is missing string field ${field}`)
+}
+
+function isConflictMessage(message: string): boolean {
+	const normalized = message.toLowerCase()
+	return normalized.includes('already exists') || normalized.includes('path conflict')
+}
+
+function isManagedSharedWorkspacePath(path: string): boolean {
+	return managedSharedWorkspacePrefixes.some((prefix) => path.startsWith(prefix))
+}

ai_evals/adapters/frontend/benchmarkRunner.ts

@@ -1,4 +1,5 @@
 import { loadSelectedCases } from "../../core/cases";
+import { resolveBackendValidationSettings } from "../../core/backendValidation";
 import {
   formatRunModelLabel,
   getFrontendEvalModel,
@@ -22,9 +23,13 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
   const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1";
   const verbose = process.env.WMILL_FRONTEND_AI_EVAL_VERBOSE === "1";
   const model = resolveEvalModel(mode, process.env.WMILL_FRONTEND_AI_EVAL_MODEL);
+  const backendValidation = resolveBackendValidationSettings({
+    evalMode: mode,
+    requestedMode: process.env.WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION,
+  });
 
   const selectedCases = await loadSelectedCases(mode, caseIds);
-  const modeRunner = getModeRunner(mode, getFrontendEvalModel(model));
+  const modeRunner = getModeRunner(mode, getFrontendEvalModel(model), backendValidation);
   const runModel = formatRunModelLabel(mode, model);
   const caseResults = await runSuite({
     modeRunner,
@@ -48,15 +53,16 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
 
 function getModeRunner(
   mode: FrontendBenchmarkMode,
-  model: ReturnType<typeof getFrontendEvalModel>
+  model: ReturnType<typeof getFrontendEvalModel>,
+  backendValidation: ReturnType<typeof resolveBackendValidationSettings>
 ): ModeRunner<any, any, any> {
   switch (mode) {
     case "flow":
-      return createFlowModeRunner(model);
+      return createFlowModeRunner(model, backendValidation);
     case "app":
       return createAppModeRunner(model);
     case "script":
-      return createScriptModeRunner(model);
+      return createScriptModeRunner(model, backendValidation);
   }
 }
 

ai_evals/adapters/frontend/runtime.ts

@@ -22,6 +22,7 @@ export async function runFrontendBenchmarkAdapter(input: {
 	runs: number
 	model?: string
 	verbose?: boolean
+	backendValidation?: string
 }): Promise<BenchmarkRunResult> {
 	const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-'))
 	const outputPath = path.join(tempDir, 'result.json')
@@ -48,7 +49,8 @@ export async function runFrontendBenchmarkAdapter(input: {
 					WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs),
 					WMILL_FRONTEND_AI_EVAL_MODEL: input.model ?? "",
 					WMILL_FRONTEND_AI_EVAL_PROGRESS: '1',
-					WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0'
+					WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0',
+					WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? ''
 				}
 			}
 		)

ai_evals/cases/flow.yaml

@@ -3,6 +3,11 @@
     Create a flow that takes two numbers, `a` and `b`, and returns their sum.
     Keep it simple and use a single step named `sum_numbers`.
   expected: ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json
+  runtime:
+    backendPreview:
+      args:
+        a: 4
+        b: 5
   judgeChecklist:
     - "the flow takes `a` and `b` as inputs"
     - "the main step is named `sum_numbers`"
@@ -15,6 +20,11 @@
     The flow should take `a` and `b` as inputs and use a single step named `sum_numbers`.
   initial: ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json
   expected: ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json
+  runtime:
+    backendPreview:
+      args:
+        a: 2
+        b: 3
   judgeChecklist:
     - "the flow takes `a` and `b` as inputs"
     - "the main step is named `sum_numbers`"
@@ -27,6 +37,11 @@
     Use a single step named `call_add_numbers`.
   initial: ai_evals/fixtures/frontend/flow/initial/test2_call_existing_subflow_initial.json
   expected: ai_evals/fixtures/frontend/flow/expected/test2_call_existing_subflow.json
+  runtime:
+    backendPreview:
+      args:
+        a: 7
+        b: 8
   judgeChecklist:
     - "the parent flow takes `a` and `b` as inputs"
     - "the main step is named `call_add_numbers`"

ai_evals/cli/index.ts

@@ -2,6 +2,10 @@
 
 import { Command, InvalidArgumentError } from "commander";
 import { loadCases, loadSelectedCases } from "../core/cases";
+import {
+  BACKEND_VALIDATION_MODES,
+  parseBackendValidationMode,
+} from "../core/backendValidation";
 import {
   EVAL_MODELS,
   type EvalModelSpec,
@@ -43,6 +47,7 @@ async function main() {
         "  bun run cli -- run flow --models haiku,opus,4o",
         "  bun run cli -- run flow flow-test0-sum-two-numbers --verbose",
         "  bun run cli -- run flow --record",
+        "  bun run cli -- run flow --backend-validation preview",
         "  bun run cli -- run flow flow-test5-simple-modification --runs 3",
         "  bun run cli -- run cli bun-hello-script",
         "",
@@ -77,6 +82,10 @@ async function main() {
     .option("--models <names>", "comma-separated model aliases to run sequentially")
     .option("--verbose", "stream assistant output during frontend runs")
     .option("--record", "append a compact summary line to ai_evals/history/<mode>.jsonl")
+    .option(
+      "--backend-validation <mode>",
+      `backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`
+    )
     .action(
       async (
         mode: EvalMode,
@@ -88,6 +97,7 @@ async function main() {
           models?: string;
           verbose?: boolean;
           record?: boolean;
+          backendValidation?: string;
         }
       ) => {
         await handleRun({
@@ -99,6 +109,7 @@ async function main() {
           models: options.models,
           verbose: options.verbose ?? false,
           record: options.record ?? false,
+          backendValidation: options.backendValidation,
         });
       }
     );
@@ -143,6 +154,7 @@ async function handleRun(input: {
   models?: string;
   verbose: boolean;
   record: boolean;
+  backendValidation?: string;
 }) {
   if (input.record && input.caseIds.length > 0) {
     throw new Error("--record only supports full-suite runs; omit case ids to record history");
@@ -153,9 +165,15 @@ async function handleRun(input: {
 
   const selectedCases = await loadSelectedCases(input.mode, input.caseIds);
   const models = resolveRequestedModels(input.mode, input.model, input.models);
+  const backendValidation = parseBackendValidationMode(
+    input.backendValidation ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION
+  );
   if (input.outputPath && models.length > 1) {
     throw new Error("--output only supports a single model run");
   }
+  if (backendValidation !== "off" && input.mode !== "flow" && input.mode !== "script") {
+    throw new Error("--backend-validation currently supports only flow and script modes");
+  }
 
   const summaries: Array<{ label: string; passRate: number; averageDurationMs: number }> = [];
 
@@ -177,6 +195,7 @@ async function handleRun(input: {
             runs: input.runs,
             model: model.id,
             verbose: input.verbose,
+            backendValidation,
           });
 
     const resolvedOutputPath =

ai_evals/core/backendValidation.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from "bun:test";
+import {
+  parseBackendValidationMode,
+  resolveBackendValidationSettings,
+} from "./backendValidation";
+
+describe("parseBackendValidationMode", () => {
+  it("defaults to off", () => {
+    expect(parseBackendValidationMode(undefined)).toBe("off");
+    expect(parseBackendValidationMode("0")).toBe("off");
+    expect(parseBackendValidationMode("false")).toBe("off");
+  });
+
+  it("accepts preview aliases", () => {
+    expect(parseBackendValidationMode("preview")).toBe("preview");
+    expect(parseBackendValidationMode("1")).toBe("preview");
+    expect(parseBackendValidationMode("true")).toBe("preview");
+  });
+
+  it("rejects unknown modes", () => {
+    expect(() => parseBackendValidationMode("maybe")).toThrow(
+      "Unsupported backend validation mode: maybe"
+    );
+  });
+});
+
+describe("resolveBackendValidationSettings", () => {
+  it("rejects unsupported eval modes", () => {
+    expect(() =>
+      resolveBackendValidationSettings({
+        evalMode: "app",
+        requestedMode: "preview",
+      })
+    ).toThrow('Backend validation mode "preview" is only supported for flow and script evals');
+  });
+});

ai_evals/core/backendValidation.ts

@@ -0,0 +1,104 @@
+import type { EvalMode } from "./types";
+
+export const BACKEND_VALIDATION_MODES = ["off", "preview"] as const;
+
+export type BackendValidationMode = (typeof BACKEND_VALIDATION_MODES)[number];
+
+export interface BackendValidationSettings {
+  mode: BackendValidationMode;
+  baseUrl: string;
+  email: string;
+  password: string;
+  keepWorkspaces: boolean;
+  workspaceOverride?: string;
+  workspacePrefix: string;
+  pollIntervalMs: number;
+  maxWaitMs: number;
+}
+
+export function parseBackendValidationMode(value?: string | null): BackendValidationMode {
+  const normalized = value?.trim().toLowerCase();
+
+  if (!normalized || normalized === "off" || normalized === "false" || normalized === "0") {
+    return "off";
+  }
+
+  if (normalized === "preview" || normalized === "true" || normalized === "1") {
+    return "preview";
+  }
+
+  throw new Error(
+    `Unsupported backend validation mode: ${value}. Use one of: ${BACKEND_VALIDATION_MODES.join(", ")}`
+  );
+}
+
+export function resolveBackendValidationSettings(input: {
+  evalMode: EvalMode;
+  requestedMode?: string | null;
+}): BackendValidationSettings {
+  const mode = parseBackendValidationMode(
+    input.requestedMode ?? process.env.WMILL_AI_EVAL_BACKEND_VALIDATION
+  );
+
+  if (mode !== "off" && input.evalMode !== "flow" && input.evalMode !== "script") {
+    throw new Error(
+      `Backend validation mode "${mode}" is only supported for flow and script evals`
+    );
+  }
+
+  return {
+    mode,
+    baseUrl: normalizeBaseUrl(
+      process.env.WMILL_AI_EVAL_BACKEND_URL ??
+        process.env.WINDMILL_URL ??
+        process.env.WINDMILL_BASE_URL ??
+        process.env.REMOTE ??
+        "http://127.0.0.1:8000"
+    ),
+    email: process.env.WMILL_AI_EVAL_BACKEND_EMAIL ?? "admin@windmill.dev",
+    password: process.env.WMILL_AI_EVAL_BACKEND_PASSWORD ?? "changeme",
+    keepWorkspaces: isTruthy(process.env.WMILL_AI_EVAL_KEEP_WORKSPACES),
+    workspaceOverride: sanitizeOptionalWorkspaceId(process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE),
+    workspacePrefix: sanitizeWorkspacePrefix(
+      process.env.WMILL_AI_EVAL_WORKSPACE_PREFIX ?? "ai-evals"
+    ),
+    pollIntervalMs: parsePositiveInteger(
+      process.env.WMILL_AI_EVAL_BACKEND_POLL_INTERVAL_MS,
+      2000
+    ),
+    maxWaitMs: parsePositiveInteger(process.env.WMILL_AI_EVAL_BACKEND_MAX_WAIT_MS, 120000),
+  };
+}
+
+function normalizeBaseUrl(value: string): string {
+  return value.replace(/\/+$/, "");
+}
+
+function sanitizeWorkspacePrefix(value: string): string {
+  const sanitized = value
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9-]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  return sanitized.length > 0 ? sanitized : "ai-evals";
+}
+
+function sanitizeOptionalWorkspaceId(value: string | undefined): string | undefined {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed : undefined;
+}
+
+function isTruthy(value: string | undefined): boolean {
+  if (!value) {
+    return false;
+  }
+  return ["1", "true", "yes", "on"].includes(value.trim().toLowerCase());
+}
+
+function parsePositiveInteger(value: string | undefined, fallback: number): number {
+  if (!value) {
+    return fallback;
+  }
+  const parsed = Number(value);
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}

ai_evals/core/cases.test.ts

@@ -0,0 +1,18 @@
+import { describe, expect, it } from "bun:test";
+import { loadCases } from "./cases";
+
+describe("loadCases", () => {
+  it("loads backend preview runtime config for opt-in flow cases", async () => {
+    const flowCases = await loadCases("flow");
+    const caseEntry = flowCases.find((entry) => entry.id === "flow-test1-reuse-existing-script");
+
+    expect(caseEntry?.runtime).toEqual({
+      backendPreview: {
+        args: {
+          a: 2,
+          b: 3,
+        },
+      },
+    });
+  });
+});

ai_evals/core/cases.ts

@@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { parse } from "yaml";
-import type { EvalCase, EvalMode, FlowValidationSpec } from "./types";
+import type { EvalCase, EvalCaseRuntimeSpec, EvalMode, FlowValidationSpec } from "./types";
 
 const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url));
 const CASES_DIR = path.join(REPO_ROOT, "ai_evals", "cases");
@@ -14,6 +14,7 @@ interface RawEvalCase {
   expected?: string;
   validate?: FlowValidationSpec;
   judgeChecklist?: string[];
+  runtime?: EvalCaseRuntimeSpec;
 }
 
 export function getRepoRoot(): string {
@@ -40,6 +41,7 @@ export async function loadCases(mode: EvalMode): Promise<EvalCase[]> {
     expectedPath: resolveFixturePath(entry.expected),
     validate: entry.validate,
     judgeChecklist: entry.judgeChecklist,
+    runtime: entry.runtime,
   }));
 }
 

ai_evals/core/runSuite.ts

@@ -155,6 +155,44 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
           run,
         }),
       ];
+      const artifactFiles = input.modeRunner.buildArtifacts?.(run.actual) ?? [];
+
+      if (run.success && input.modeRunner.backendValidate) {
+        try {
+          const backendValidation = await input.modeRunner.backendValidate({
+            evalCase: input.evalCase,
+            prompt: input.evalCase.prompt,
+            initial,
+            expected,
+            actual: run.actual,
+            run,
+            context: {
+              caseId: input.evalCase.id,
+              caseNumber: input.caseIndex + 1,
+              totalCases: input.totalCases,
+              attempt,
+              runs: input.runs,
+              verbose: input.verbose,
+              onAssistantMessageStart: undefined,
+              onAssistantChunk: undefined,
+              onAssistantMessageEnd: undefined,
+            },
+          });
+
+          if (backendValidation) {
+            checks.push(...backendValidation.checks);
+            artifactFiles.push(...(backendValidation.artifactFiles ?? []));
+          }
+        } catch (error) {
+          checks.push(
+            buildCheck(
+              "backend validation succeeded",
+              false,
+              error instanceof Error ? error.message : String(error)
+            )
+          );
+        }
+      }
 
       let judgeScore: number | null = null;
       let judgeSummary: string | null = null;
@@ -182,7 +220,6 @@ async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
         );
       }
 
-      const artifactFiles = input.modeRunner.buildArtifacts?.(run.actual) ?? [];
       const attemptResult: BenchmarkAttemptResult = {
         attempt,
         passed: checks.every((check) => check.passed),

ai_evals/core/types.ts

@@ -2,6 +2,15 @@ export const EVAL_MODES = ["cli", "flow", "script", "app"] as const;
 
 export type EvalMode = (typeof EVAL_MODES)[number];
 
+export interface EvalCaseRuntimeBackendPreview {
+  args?: Record<string, unknown>;
+  timeoutSeconds?: number;
+}
+
+export interface EvalCaseRuntimeSpec {
+  backendPreview?: EvalCaseRuntimeBackendPreview;
+}
+
 export interface FlowValidationSpec {
   schemaRequiredPaths?: string[];
   schemaAnyOf?: Array<{
@@ -23,6 +32,7 @@ export interface EvalCase {
   expectedPath?: string;
   validate?: FlowValidationSpec;
   judgeChecklist?: string[];
+  runtime?: EvalCaseRuntimeSpec;
 }
 
 export interface BenchmarkCheck {
@@ -43,6 +53,11 @@ export interface BenchmarkArtifactFile {
   content: string;
 }
 
+export interface BackendValidationResult {
+  checks: BenchmarkCheck[];
+  artifactFiles?: BenchmarkArtifactFile[];
+}
+
 export interface BenchmarkTokenUsage {
   prompt: number;
   completion: number;
@@ -91,6 +106,15 @@ export interface ModeRunner<TInitial, TExpected, TActual> {
     actual: TActual;
     run: ModeRunOutput<TActual>;
   }): BenchmarkCheck[];
+  backendValidate?(input: {
+    evalCase: EvalCase;
+    prompt: string;
+    initial: TInitial | undefined;
+    expected: TExpected | undefined;
+    actual: TActual;
+    run: ModeRunOutput<TActual>;
+    context: ModeRunContext;
+  }): Promise<BackendValidationResult | null>;
   buildArtifacts?(actual: TActual): BenchmarkArtifactFile[];
 }
 

ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json

@@ -5,7 +5,7 @@
         "id": "sum_numbers",
         "value": {
           "type": "script",
-          "path": "f/evals/add_two_numbers.ts",
+          "path": "f/evals/add_two_numbers",
           "input_transforms": {
             "a": {
               "type": "javascript",

ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json

@@ -2,7 +2,7 @@
   "workspace": {
     "scripts": [
       {
-        "path": "f/evals/add_two_numbers.ts",
+        "path": "f/evals/add_two_numbers",
         "summary": "Add two numbers",
         "description": "Returns the sum of two numeric inputs.",
         "language": "bun",

ai_evals/modes/flow.ts

@@ -1,4 +1,5 @@
 import { readJsonFile } from "../core/files";
+import type { BackendValidationSettings } from "../core/backendValidation";
 import type { FrontendEvalModelConfig } from "../core/models";
 import { validateFlowState, type FlowState } from "../core/validators";
 import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
@@ -7,6 +8,7 @@ import {
   type FlowFixture,
 } from "../adapters/frontend/core/flow/flowEvalRunner";
 import type { FlowWorkspaceFixtures } from "../adapters/frontend/core/flow/fileHelpers";
+import { BackendPreviewClient } from "../adapters/frontend/backendPreview";
 import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon";
 
 interface FlowInitialFixture {
@@ -15,7 +17,8 @@ interface FlowInitialFixture {
 }
 
 export function createFlowModeRunner(
-  modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL
+  modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
+  backendValidation?: BackendValidationSettings
 ): ModeRunner<FlowInitialFixture, FlowState, FlowState> {
   return {
     mode: "flow",
@@ -61,6 +64,65 @@ export function createFlowModeRunner(
         validate: evalCase.validate,
       });
     },
+    async backendValidate({ evalCase, initial, actual, context }) {
+      if (backendValidation?.mode !== "preview" || !evalCase.runtime?.backendPreview) {
+        return null;
+      }
+
+      if (!actual.value) {
+        return {
+          checks: [
+            {
+              name: "backend flow preview succeeded",
+              passed: false,
+              details: "Generated flow is missing value.modules",
+            },
+          ],
+        };
+      }
+
+      const previewClient = new BackendPreviewClient(backendValidation);
+      return await previewClient.withWorkspace(evalCase.id, context.attempt, async (workspaceId) => {
+        await seedWorkspaceFixtures(previewClient, workspaceId, initial?.workspace);
+
+        const completedJob = await previewClient.runFlowPreview({
+          workspaceId,
+          value: actual.value as Record<string, unknown>,
+          args: evalCase.runtime?.backendPreview?.args ?? {},
+          timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds,
+        });
+
+        return {
+          checks: [
+            {
+              name: "backend flow preview succeeded",
+              passed: completedJob.success,
+              details: completedJob.success
+                ? `workspace=${workspaceId}`
+                : `workspace=${workspaceId}; job=${completedJob.id}`,
+            },
+          ],
+          artifactFiles: [
+            {
+              path: "backend-preview.json",
+              content:
+                JSON.stringify(
+                  {
+                    workspaceId,
+                    jobId: completedJob.id,
+                    success: completedJob.success,
+                    result: completedJob.result,
+                    logs: completedJob.logs,
+                    completedJob: completedJob.raw,
+                  },
+                  null,
+                  2
+                ) + "\n",
+            },
+          ],
+        };
+      });
+    },
     buildArtifacts(actual): BenchmarkArtifactFile[] {
       return [
         {
@@ -102,3 +164,32 @@ function normalizeFlowStateFixture(value: unknown): FlowState {
 function isObject(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
+
+async function seedWorkspaceFixtures(
+  previewClient: BackendPreviewClient,
+  workspaceId: string,
+  fixtures?: FlowWorkspaceFixtures
+): Promise<void> {
+  for (const script of fixtures?.scripts ?? []) {
+    await previewClient.createScript({
+      workspaceId,
+      path: script.path,
+      summary: script.summary,
+      description: script.description,
+      schema: script.schema,
+      content: script.content,
+      language: script.language,
+    });
+  }
+
+  for (const flow of fixtures?.flows ?? []) {
+    await previewClient.createFlow({
+      workspaceId,
+      path: flow.path,
+      summary: flow.summary,
+      description: flow.description,
+      schema: flow.schema,
+      value: flow.value as Record<string, unknown>,
+    });
+  }
+}

ai_evals/modes/script.ts

@@ -1,13 +1,16 @@
 import { readJsonFile } from "../core/files";
+import type { BackendValidationSettings } from "../core/backendValidation";
 import type { FrontendEvalModelConfig } from "../core/models";
 import { validateScriptState } from "../core/validators";
 import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
+import { BackendPreviewClient } from "../adapters/frontend/backendPreview";
 import { runScriptEval } from "../adapters/frontend/core/script/scriptEvalRunner";
 import type { ScriptEvalState } from "../adapters/frontend/core/script/fileHelpers";
 import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon";
 
 export function createScriptModeRunner(
-  modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL
+  modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
+  backendValidation?: BackendValidationSettings
 ): ModeRunner<ScriptEvalState, ScriptEvalState, ScriptEvalState> {
   return {
     mode: "script",
@@ -45,6 +48,57 @@ export function createScriptModeRunner(
     validate({ actual, initial, expected }) {
       return validateScriptState({ actual, initial, expected });
     },
+    async backendValidate({ evalCase, initial, actual, context }) {
+      if (backendValidation?.mode !== "preview") {
+        return null;
+      }
+
+      const previewClient = new BackendPreviewClient(backendValidation);
+      return await previewClient.withWorkspace(evalCase.id, context.attempt, async (workspaceId) => {
+        const completedJob = await previewClient.runScriptPreview({
+          workspaceId,
+          content: actual.code,
+          args:
+            (evalCase.runtime?.backendPreview?.args as Record<string, unknown> | undefined) ??
+            actual.args ??
+            initial?.args ??
+            {},
+          language: normalizePreviewLanguage(actual.lang),
+          path: toPreviewScriptPath(actual.path),
+          timeoutSeconds: evalCase.runtime?.backendPreview?.timeoutSeconds,
+        });
+
+        return {
+          checks: [
+            {
+              name: "backend script preview succeeded",
+              passed: completedJob.success,
+              details: completedJob.success
+                ? `workspace=${workspaceId}`
+                : `workspace=${workspaceId}; job=${completedJob.id}`,
+            },
+          ],
+          artifactFiles: [
+            {
+              path: "backend-preview.json",
+              content:
+                JSON.stringify(
+                  {
+                    workspaceId,
+                    jobId: completedJob.id,
+                    success: completedJob.success,
+                    result: completedJob.result,
+                    logs: completedJob.logs,
+                    completedJob: completedJob.raw,
+                  },
+                  null,
+                  2
+                ) + "\n",
+            },
+          ],
+        };
+      });
+    },
     buildArtifacts(actual): BenchmarkArtifactFile[] {
       return [
         {
@@ -59,3 +113,11 @@ export function createScriptModeRunner(
     },
   };
 }
+
+function normalizePreviewLanguage(language: ScriptEvalState["lang"]): string {
+  return language === "bunnative" ? "bun" : language;
+}
+
+function toPreviewScriptPath(filePath: string): string {
+  return filePath.replace(/\.[^.\/]+$/, "");
+}