kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: rewrite ai eval benchmark runner

Browse Source 
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
centdix
2026-04-08 14:18:44 +02:00
parent d176983c21
commit 550bcbcff3
59 changed files with 1985 additions and 5076 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ai_evals/.gitignore vendored
View File

@@ -1 +1,2 @@
.env
.env
results/

78
ai_evals/README.md
View File

@@ -1,61 +1,69 @@
# AI Evals
Repo-level benchmark suite for the Windmill CLI guidance and the frontend AI
chat surfaces.
Minimal benchmark runner for the four Windmill AI generation modes:
## What It Is For
- `cli`
- `flow`
- `script`
- `app`
The default workflow is:
Each case is just:
1. run the benchmark on the current checkout
2. save the local result JSON
3. make your change
4. run again
5. diff the two result files
6. optionally promote the good run into official history
- a `prompt`
- an optional `initial` fixture
- an optional `expected` fixture
This keeps local development focused on before/after comparisons instead of
named prompt variants.
Each attempt runs:
## Entry Point
1. the real production prompt/tool/guidance path
2. deterministic validation
3. LLM judging
Install once:
## Install
```bash
cd ai_evals
bun install
```
Main commands:
Frontend runs also require frontend dependencies:
```bash
cd frontend
bun install
```
## CLI
List cases:
```bash
cd ai_evals
bun run cli -- list-cases
bun run cli -- run --surface flow --runs 3
bun run cli -- diff-results ai_evals/results/before.json ai_evals/results/after.json
bun run cli -- history --limit 10
bun run cli -- cases
bun run cli -- cases flow
```
## Surfaces
Run a mode:
- `cli`: benchmark the CLI skills / AGENTS / CLAUDE guidance path
- `flow`: benchmark frontend flow chat
- `app`: benchmark frontend app chat
- `script`: benchmark frontend script chat
```bash
cd ai_evals
bun run cli -- run flow
bun run cli -- run flow flow-test5-simple-modification --runs 3
bun run cli -- run cli bun-hello-script
```
`run` always writes a JSON result file under `ai_evals/results/` unless you pass
`--output`.
## Layout
- `cli/`: benchmark CLI entrypoint
- `cases/`: eval manifests
- `fixtures/`: initial and expected frontend artifacts
- `history/`: tracked official benchmark history
- `results/`: local run outputs written by `run` and ignored by git
- `cases/`: one JSON file per mode
- `fixtures/`: initial and expected fixtures
- `core/`: shared case loading, validation, judging, and result writing
- `modes/`: one runner per mode
## Notes
- Frontend runs reuse the production frontend chat code through the Vitest
adapter under `ai_evals/adapters/frontend/`.
- CLI runs create an isolated workspace and write the current checkout's
guidance into it before running the benchmark prompt.
- Official history is separate from local experimentation. Use
`bun run cli -- promote-result ...` only for runs you want to preserve.
- Frontend modes reuse the production frontend chat code through the Vitest bridge.
- CLI mode creates an isolated workspace, writes the current checkout guidance into it, and benchmarks the real skills / AGENTS flow.
- Frontend progress streams live while the benchmark is running.

194
ai_evals/adapters/cli/artifact-eval.ts
View File

@@ -1,194 +0,0 @@
import { existsSync } from "fs";
import { mkdtemp, mkdir, readFile, rm, writeFile } from "fs/promises";
import { tmpdir } from "os";
import { dirname, join } from "path";
import { writeAiGuidanceFiles } from "../../../cli/src/guidance/writer.ts";
import { getGeneratedSkillsSource } from "./runtime";
import {
loadEvalCases,
type CliExpectedFileCheck
} from "../shared/evalCases";
import {
validateCliArtifact,
type BenchmarkCheck
} from "../shared/validators";
import {
runPromptAndCapture,
type PromptRunResult,
} from "./runtime";
export type ExpectedFile = CliExpectedFileCheck;
export interface CliArtifactEvalCase {
id: string;
description?: string;
prompt: string;
maxTurns?: number;
expectedSkill?: string;
expectedOutputSubstrings?: string[];
expectedFiles: ExpectedFile[];
}
export type ArtifactCheck = BenchmarkCheck;
export interface FileArtifactResult {
path: string;
exists: boolean;
content?: string;
}
export interface CliArtifactEvalResult {
workspaceDir: string;
renderedPrompt: string;
run: PromptRunResult;
checks: ArtifactCheck[];
expectedFiles: FileArtifactResult[];
passed: boolean;
guidanceLabel: string;
}
export interface CliGuidanceConfig {
label: string;
skillsSourcePath?: string;
agentsSourcePath?: string;
claudeSourcePath?: string;
}
const CLAUDE_PROJECT_PREAMBLE = [
"Follow the project instructions from AGENTS.md exactly.",
"Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.",
"Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
"Do not skip the Skill step."
].join(" ");
export async function loadCliArtifactEvalCases(): Promise<CliArtifactEvalCase[]> {
return loadEvalCases("cli").map((entry) => ({
id: entry.id,
description: entry.title,
prompt: entry.userPrompt,
maxTurns:
typeof entry.workspaceContext.max_turns === "number"
? entry.workspaceContext.max_turns
: undefined,
expectedSkill: entry.artifactChecks.expectedSkill,
expectedOutputSubstrings: entry.artifactChecks.expectedOutputSubstrings,
expectedFiles: entry.artifactChecks.expectedFiles
}));
}
export async function runCliArtifactEvalCase(
evalCase: CliArtifactEvalCase,
options: {
guidance: CliGuidanceConfig;
}
): Promise<CliArtifactEvalResult> {
const workspaceDir = await createIsolatedWorkspace(evalCase.id, options.guidance);
try {
const renderedPrompt = await renderPrompt(evalCase.prompt, workspaceDir);
const run = await runPromptAndCapture(
renderedPrompt,
workspaceDir,
evalCase.maxTurns ?? 6
);
const fileResults = await collectExpectedFiles(workspaceDir, evalCase.expectedFiles);
const checks = buildChecks(evalCase, run, fileResults);
return {
workspaceDir,
renderedPrompt,
run,
checks,
expectedFiles: fileResults,
passed: checks.every((check) => check.required === false || check.passed),
guidanceLabel: options.guidance.label
};
} catch (error) {
if (!shouldKeepWorkspace()) {
await cleanupWorkspace(workspaceDir);
}
throw error;
}
}
export async function cleanupWorkspace(workspaceDir: string): Promise<void> {
await rm(workspaceDir, { recursive: true, force: true });
}
export function shouldKeepWorkspace(): boolean {
return process.env.WMILL_CLI_EVAL_KEEP_WORKSPACE === "1";
}
async function createIsolatedWorkspace(
caseId: string,
guidance: CliGuidanceConfig
): Promise<string> {
const workspaceDir = await mkdtemp(join(tmpdir(), `wmill-cli-artifact-${caseId}-`));
await mkdir(dirname(join(workspaceDir, ".claude", "skills")), { recursive: true });
await writeAiGuidanceFiles({
targetDir: workspaceDir,
nonDottedPaths: true,
overwriteProjectGuidance: true,
skillsSourcePath: guidance.skillsSourcePath ?? getGeneratedSkillsSource(),
agentsSourcePath: guidance.agentsSourcePath,
claudeSourcePath: guidance.claudeSourcePath,
});
await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8");
return workspaceDir;
}
async function renderPrompt(prompt: string, workspaceDir: string): Promise<string> {
const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir);
const agentsInstructions = await readFile(join(workspaceDir, "AGENTS.md"), "utf8");
return [
"# Project Instructions",
agentsInstructions.trim(),
"",
"# Benchmark Harness",
CLAUDE_PROJECT_PREAMBLE,
"",
"# User Request",
renderedUserPrompt
].join("\n");
}
async function collectExpectedFiles(
workspaceDir: string,
expectedFiles: ExpectedFile[]
): Promise<FileArtifactResult[]> {
const results: FileArtifactResult[] = [];
for (const expectedFile of expectedFiles) {
const absolutePath = join(workspaceDir, expectedFile.path);
const exists = existsSync(absolutePath);
if (!exists) {
results.push({ path: expectedFile.path, exists: false });
continue;
}
results.push({
path: expectedFile.path,
exists: true,
content: await readFile(absolutePath, "utf8")
});
}
return results;
}
function buildChecks(
evalCase: CliArtifactEvalCase,
run: PromptRunResult,
fileResults: FileArtifactResult[]
): ArtifactCheck[] {
return validateCliArtifact({
assistantOutput: run.output,
skillsInvoked: run.skillsInvoked,
expectedSkill: evalCase.expectedSkill,
expectedOutputSubstrings: evalCase.expectedOutputSubstrings,
expectedFiles: evalCase.expectedFiles,
fileResults
});
}

501
ai_evals/adapters/frontend/benchmarkRunner.ts
View File

@@ -1,452 +1,75 @@
import type { AIProvider } from '$lib/gen/types.gen'
import { loadAppEvalCases, loadFlowEvalCases, loadScriptEvalCases } from './core/evalCaseLoader'
import type { VariantConfig } from './core/shared'
import {
allRequiredChecksPassed,
buildJudgeChecks,
getRequiredFailedChecks,
requiredCheck,
validateAppArtifact,
validateFlowArtifact,
validateScriptArtifact,
type BenchmarkCheck
} from '../shared/validators'
import { loadSelectedCases } from "../../core/cases";
import { buildRunResult } from "../../core/results";
import { runSuite } from "../../core/runSuite";
import type { BenchmarkRunResult, ModeRunner } from "../../core/types";
import { emitFrontendBenchmarkProgress } from "./progress";
import { createAppModeRunner } from "../../modes/app";
import { createFlowModeRunner } from "../../modes/flow";
import { createScriptModeRunner } from "../../modes/script";
import { DEFAULT_JUDGE_MODEL } from "../../core/judge";
import { getFrontendRunModelLabel } from "../../modes/frontendCommon";
export type FrontendBenchmarkSurface = 'flow' | 'app' | 'script'
export type FrontendBenchmarkMode = "flow" | "app" | "script";
export interface FrontendBenchmarkConfig {
provider?: AIProvider
model?: string
systemPrompt?: {
mode: 'append' | 'replace'
content: string
}
export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult> {
const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE);
const caseIds = parseOptionalJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS);
const runs = parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, "WMILL_FRONTEND_AI_EVAL_RUNS");
const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1";
const selectedCases = await loadSelectedCases(mode, caseIds);
const modeRunner = getModeRunner(mode);
const caseResults = await runSuite({
modeRunner,
cases: selectedCases,
runs,
runModel: getFrontendRunModelLabel(),
judgeModel: DEFAULT_JUDGE_MODEL,
onProgress: emitProgress ? (event) => emitFrontendBenchmarkProgress(event) : undefined,
});
return buildRunResult({
mode,
runs,
runModel: getFrontendRunModelLabel(),
judgeModel: DEFAULT_JUDGE_MODEL,
caseResults,
});
}
export interface FrontendBenchmarkAttempt {
attempt: number
passed: boolean
durationMs: number
assistantMessageCount: number
toolCallCount: number
toolsUsed: string[]
checks: BenchmarkCheck[]
requiredFailedChecks: string[]
judgeScore: number | null
judgeStatement: string | null
error: string | null
function getModeRunner(mode: FrontendBenchmarkMode): ModeRunner<any, any, any> {
switch (mode) {
case "flow":
return createFlowModeRunner();
case "app":
return createAppModeRunner();
case "script":
return createScriptModeRunner();
}
}
export interface FrontendBenchmarkCaseResult {
caseId: string
attempts: FrontendBenchmarkAttempt[]
}
export interface FrontendBenchmarkPayload {
surface: FrontendBenchmarkSurface
runs: number
provider: AIProvider
model: string
judgeModel: string | null
caseResults: FrontendBenchmarkCaseResult[]
}
const DEFAULT_MIN_JUDGE_SCORE = 80
const DEFAULT_PROVIDER: AIProvider = 'anthropic'
const DEFAULT_MODEL = 'claude-haiku-4-5-20251001'
const FRONTEND_JUDGE_MODEL = 'claude-sonnet-4-6'
export async function runFrontendBenchmarkFromEnv(): Promise<FrontendBenchmarkPayload> {
return runFrontendBenchmark({
surface: parseSurface(process.env.WMILL_FRONTEND_AI_EVAL_SURFACE),
caseIds: parseOptionalJsonStringArray(process.env.WMILL_FRONTEND_AI_EVAL_CASE_IDS),
runs: parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, 'WMILL_FRONTEND_AI_EVAL_RUNS'),
config: parseConfig(process.env.WMILL_FRONTEND_AI_EVAL_CONFIG)
})
}
export async function runFrontendBenchmark(input: {
surface: FrontendBenchmarkSurface
caseIds: string[]
runs: number
config?: FrontendBenchmarkConfig
}): Promise<FrontendBenchmarkPayload> {
switch (input.surface) {
case 'flow':
return await runFlowBenchmark({
surface: 'flow',
caseIds: input.caseIds,
runs: input.runs,
config: input.config
})
case 'app':
return await runAppBenchmark({
surface: 'app',
caseIds: input.caseIds,
runs: input.runs,
config: input.config
})
case 'script':
return await runScriptBenchmark({
surface: 'script',
caseIds: input.caseIds,
runs: input.runs,
config: input.config
})
default:
throw new Error(`Unsupported frontend benchmark surface: ${String(input.surface)}`)
}
}
async function runFlowBenchmark(input: {
surface: 'flow'
caseIds: string[]
runs: number
config?: FrontendBenchmarkConfig
}): Promise<FrontendBenchmarkPayload> {
const { runFlowEval } = await import('./core/flow/flowEvalRunner')
const allCases = loadFlowEvalCases()
const selectedCases = resolveCases(allCases, input.caseIds, 'frontend flow')
const resolvedConfig = resolveConfig(input.config)
return {
surface: input.surface,
runs: input.runs,
provider: resolvedConfig.provider,
model: resolvedConfig.model,
judgeModel: FRONTEND_JUDGE_MODEL,
caseResults: await Promise.all(
selectedCases.map(async (testCase) => ({
caseId: testCase.id,
attempts: await runRepeated(input.runs, async (attempt) => {
const startedAt = Date.now()
const result = await runFlowEval(
testCase.userPrompt,
getApiKeyForProvider(resolvedConfig.provider),
{
initialModules: testCase.initialFlow?.value?.modules,
initialSchema: testCase.initialFlow?.schema,
expectedFlow: testCase.expectedFlow as unknown as Parameters<
typeof runFlowEval
>[2]['expectedFlow'],
variant: resolvedConfig.variant,
provider: resolvedConfig.provider,
model: resolvedConfig.model
}
)
const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE
const checks = [
requiredCheck('chat run succeeded', result.success, result.error),
...validateFlowArtifact({
generatedFlow: {
value: { modules: result.flow.value.modules },
schema: result.flow.schema
},
expectedFlow: testCase.expectedFlow
}),
...buildJudgeChecks({
evaluationResult: result.evaluationResult,
minJudgeScore
})
]
return {
attempt,
passed: allRequiredChecksPassed(checks),
durationMs: Date.now() - startedAt,
assistantMessageCount: result.iterations,
toolCallCount: result.toolCallsCount,
toolsUsed: uniqueStrings(result.toolsCalled),
checks,
requiredFailedChecks: getRequiredFailedChecks(checks),
judgeScore: result.evaluationResult?.resemblanceScore ?? null,
judgeStatement: result.evaluationResult?.statement ?? null,
error: result.error ?? result.evaluationResult?.error ?? null
} satisfies FrontendBenchmarkAttempt
})
}))
)
}
}
async function runAppBenchmark(input: {
surface: 'app'
caseIds: string[]
runs: number
config?: FrontendBenchmarkConfig
}): Promise<FrontendBenchmarkPayload> {
const { runAppEval } = await import('./core/app/appEvalRunner')
const { loadAppFixtureForEval } = await import('./core/app/appFixtureLoader')
const allCases = loadAppEvalCases()
const selectedCases = resolveCases(allCases, input.caseIds, 'frontend app')
const resolvedConfig = resolveConfig(input.config)
return {
surface: input.surface,
runs: input.runs,
provider: resolvedConfig.provider,
model: resolvedConfig.model,
judgeModel: FRONTEND_JUDGE_MODEL,
caseResults: await Promise.all(
selectedCases.map(async (testCase) => {
const fixture = testCase.initialAppFixturePath
? await loadAppFixtureForEval(testCase.initialAppFixturePath)
: { initialFrontend: {}, initialBackend: {} }
return {
caseId: testCase.id,
attempts: await runRepeated(input.runs, async (attempt) => {
const startedAt = Date.now()
const result = await runAppEval(
testCase.userPrompt,
getApiKeyForProvider(resolvedConfig.provider),
{
...fixture,
variant: resolvedConfig.variant,
provider: resolvedConfig.provider,
model: resolvedConfig.model
}
)
const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE
const checks = [
requiredCheck('chat run succeeded', result.success, result.error),
...validateAppArtifact({
generatedApp: result.files,
initialApp: testCase.initialAppFixturePath
? {
frontend: fixture.initialFrontend,
backend: fixture.initialBackend
}
: undefined
}),
...buildJudgeChecks({
evaluationResult: result.evaluationResult,
minJudgeScore
})
]
return {
attempt,
passed: allRequiredChecksPassed(checks),
durationMs: Date.now() - startedAt,
assistantMessageCount: result.iterations,
toolCallCount: result.toolCallsCount,
toolsUsed: uniqueStrings(result.toolsCalled),
checks,
requiredFailedChecks: getRequiredFailedChecks(checks),
judgeScore: result.evaluationResult?.resemblanceScore ?? null,
judgeStatement: result.evaluationResult?.statement ?? null,
error: result.error ?? result.evaluationResult?.error ?? null
} satisfies FrontendBenchmarkAttempt
})
}
})
)
}
}
async function runScriptBenchmark(input: {
surface: 'script'
caseIds: string[]
runs: number
config?: FrontendBenchmarkConfig
}): Promise<FrontendBenchmarkPayload> {
const { runScriptEval } = await import('./core/script/scriptEvalRunner')
const allCases = loadScriptEvalCases()
const selectedCases = resolveCases(allCases, input.caseIds, 'frontend script')
const resolvedConfig = resolveConfig(input.config)
return {
surface: input.surface,
runs: input.runs,
provider: resolvedConfig.provider,
model: resolvedConfig.model,
judgeModel: FRONTEND_JUDGE_MODEL,
caseResults: await Promise.all(
selectedCases.map(async (testCase) => ({
caseId: testCase.id,
attempts: await runRepeated(input.runs, async (attempt) => {
const startedAt = Date.now()
const result = await runScriptEval(
testCase.userPrompt,
getApiKeyForProvider(resolvedConfig.provider),
{
initialScript: testCase.initialScript ?? testCase.expectedScript,
expectedScript: testCase.expectedScript,
variant: resolvedConfig.variant,
provider: resolvedConfig.provider,
model: resolvedConfig.model
}
)
const minJudgeScore = testCase.minJudgeScore ?? DEFAULT_MIN_JUDGE_SCORE
const checks = [
requiredCheck('chat run succeeded', result.success, result.error),
...validateScriptArtifact({
generatedScript: result.script,
expectedScript: testCase.expectedScript,
initialScript: testCase.initialScript
}),
...buildJudgeChecks({
evaluationResult: result.evaluationResult,
minJudgeScore
})
]
return {
attempt,
passed: allRequiredChecksPassed(checks),
durationMs: Date.now() - startedAt,
assistantMessageCount: result.iterations,
toolCallCount: result.toolCallsCount,
toolsUsed: uniqueStrings(result.toolsCalled),
checks,
requiredFailedChecks: getRequiredFailedChecks(checks),
judgeScore: result.evaluationResult?.resemblanceScore ?? null,
judgeStatement: result.evaluationResult?.statement ?? null,
error: result.error ?? result.evaluationResult?.error ?? null
} satisfies FrontendBenchmarkAttempt
})
}))
)
}
}
function resolveConfig(config?: FrontendBenchmarkConfig): {
provider: AIProvider
model: string
variant: VariantConfig | undefined
} {
const provider = config?.provider ?? DEFAULT_PROVIDER
const model = config?.model ?? DEFAULT_MODEL
if (!config?.systemPrompt) {
return {
provider,
model,
variant: undefined
}
}
return {
provider,
model,
variant: {
name: config.systemPrompt.mode === 'replace' ? 'custom-system-prompt' : 'appended-system-prompt',
systemPrompt:
config.systemPrompt.mode === 'replace'
? { type: 'custom', content: config.systemPrompt.content }
: { type: 'default-with-custom', custom: config.systemPrompt.content },
tools: { type: 'default' },
model
}
}
}
function resolveCases<T extends { id: string }>(
allCases: T[],
caseIds: string[],
surfaceLabel: string
): T[] {
if (caseIds.length === 0) {
return allCases
}
return caseIds.map((caseId) => {
const testCase = allCases.find((entry) => entry.id === caseId)
if (!testCase) {
throw new Error(`Unknown ${surfaceLabel} case: ${caseId}`)
}
return testCase
})
}
function parseConfig(value: string | undefined): FrontendBenchmarkConfig | undefined {
if (!value) {
return undefined
}
const parsed = JSON.parse(value) as FrontendBenchmarkConfig
if (!parsed || typeof parsed !== 'object') {
throw new Error('WMILL_FRONTEND_AI_EVAL_CONFIG must be a JSON object')
}
if (parsed.provider && parsed.provider !== 'anthropic' && parsed.provider !== 'openai') {
throw new Error('WMILL_FRONTEND_AI_EVAL_CONFIG.provider must be "anthropic" or "openai"')
}
if (parsed.systemPrompt) {
const systemPrompt = parsed.systemPrompt
if (
(systemPrompt.mode !== 'append' && systemPrompt.mode !== 'replace') ||
typeof systemPrompt.content !== 'string' ||
systemPrompt.content.trim().length === 0
) {
throw new Error(
'WMILL_FRONTEND_AI_EVAL_CONFIG.systemPrompt must include mode "append" or "replace" and non-empty content'
)
}
}
return parsed
function parseMode(value: string | undefined): FrontendBenchmarkMode {
if (value === "flow" || value === "app" || value === "script") {
return value;
}
throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`);
}
function parseOptionalJsonStringArray(value: string | undefined): string[] {
if (!value) {
return []
}
const parsed = JSON.parse(value) as unknown
if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== 'string')) {
throw new Error('WMILL_FRONTEND_AI_EVAL_CASE_IDS must be a JSON string array')
}
return parsed
}
function parseSurface(value: string | undefined): FrontendBenchmarkSurface {
if (value === 'flow' || value === 'app' || value === 'script') {
return value
}
throw new Error('WMILL_FRONTEND_AI_EVAL_SURFACE must be "flow", "app", or "script"')
if (!value) {
return [];
}
const parsed = JSON.parse(value) as unknown;
if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== "string")) {
throw new Error("WMILL_FRONTEND_AI_EVAL_CASE_IDS must be a JSON string array");
}
return parsed;
}
function parsePositiveInteger(value: string | undefined, envName: string): number {
const parsed = Number.parseInt(value ?? '', 10)
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${envName} must be a positive integer`)
}
return parsed
}
function getApiKeyForProvider(provider: AIProvider): string {
if (provider === 'anthropic') {
const apiKey = process.env.ANTHROPIC_API_KEY
if (!apiKey) {
throw new Error('ANTHROPIC_API_KEY is required for frontend benchmark runs')
}
return apiKey
}
if (provider === 'openai') {
const apiKey = process.env.OPENAI_API_KEY
if (!apiKey) {
throw new Error('OPENAI_API_KEY is required for frontend benchmark runs')
}
return apiKey
}
throw new Error(`Unsupported frontend benchmark provider: ${provider}`)
}
async function runRepeated<T>(runs: number, fn: (attempt: number) => Promise<T>): Promise<T[]> {
const results: T[] = []
for (let attempt = 1; attempt <= runs; attempt += 1) {
results.push(await fn(attempt))
}
return results
}
function uniqueStrings(values: string[]): string[] {
return [...new Set(values)].sort((left, right) => left.localeCompare(right))
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed <= 0) {
throw new Error(`${envName} must be a positive integer`);
}
return parsed;
}

174
ai_evals/adapters/frontend/core/app/appEvalComparison.ts
View File

@@ -1,174 +0,0 @@
import Anthropic from '@anthropic-ai/sdk'
import type {
AppFiles,
BackendRunnable
} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
import { BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared'
import type { EvaluationResult } from '../shared'
/**
* Expected app structure for evaluation.
*/
export interface ExpectedApp {
frontend: Record<string, string>
backend: Record<string, BackendRunnable>
}
/**
* Initial app state for evaluation context.
*/
export interface InitialApp {
frontend: Record<string, string>
backend: Record<string, BackendRunnable>
}
/**
* System prompt for evaluating app generation without a reference expected app.
* Evaluates based on user request fulfillment and appropriate modifications to initial state.
*/
const APP_GENERATION_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill Raw App definitions. Your task is to evaluate a generated app based on:
1. The original user request/prompt
2. The initial app state (if any) - this is what the app looked like before the AI made changes
## Windmill Raw App Context
- Raw Apps consist of frontend files and backend runnables
- Frontend files are TypeScript/JavaScript files bundled with esbuild (entrypoint: index.tsx)
- Backend runnables can be: inline scripts (TypeScript/Python), workspace scripts, workspace flows, or hub scripts
- Frontend calls backend using \`await backend.<runnable_key>(args...)\`
- Each backend runnable has a key (identifier), name (description), type, and configuration
## Backend Runnable Types
- **inline**: Custom code with \`inlineScript.language\` and \`inlineScript.content\`
- **script**: Workspace script reference with \`path\`
- **flow**: Workspace flow reference with \`path\`
- **hubscript**: Hub script reference with \`path\`
## Evaluation Criteria
1. **User Request Fulfillment**: Does the generated app address ALL requirements from the user's original prompt?
- Are all requested features implemented?
- Does the frontend UI match the requirements?
- Are the correct backend runnables created?
2. **Appropriate Modifications** (if initial app was provided):
- Were the changes made relevant to the user's request?
- Was existing functionality preserved where appropriate?
- Were only necessary changes made (no unnecessary removals or additions)?
3. **Frontend Structure**: Are the frontend files correctly organized and implemented?
- Is the code valid TypeScript/JavaScript?
- Are components properly structured?
- Are backend calls correctly made?
4. **Backend Structure**: Are the backend runnables correctly configured?
- Do inline scripts have proper main functions?
- Are types and paths correct for non-inline runnables?
5. **Integration**: Does the frontend correctly call the backend?
- Are the runnable keys correctly referenced?
- Are arguments passed correctly?
6. **Code Quality**: Is the code functionally correct and well-structured?
## Important Notes
- Focus on whether the user's request was fulfilled, not on stylistic preferences
- If an initial app was provided, evaluate the appropriateness of the changes made
- For new apps (no initial state), evaluate completeness and correctness
- Extra helper functions or slightly different approaches can still score high if they accomplish the goal
${BASE_EVALUATOR_RESPONSE_FORMAT}`
/**
* Evaluates how well a generated app fulfills the user's request, considering any initial app state.
* Uses Anthropic API directly.
*/
export async function evaluateAppGeneration(
userPrompt: string,
generatedApp: AppFiles,
initialApp?: InitialApp
): Promise<EvaluationResult> {
// @ts-ignore
const apiKey = process.env.ANTHROPIC_API_KEY
if (!apiKey) {
return {
success: false,
resemblanceScore: 0,
statement: 'No API key available for evaluation',
error: 'ANTHROPIC_API_KEY not set'
}
}
const client = new Anthropic({ apiKey })
let userMessage = `## User's Original Request
${userPrompt}
`
if (initialApp) {
userMessage += `## Initial App State (before AI modifications)
\`\`\`json
${JSON.stringify(initialApp, null, 2)}
\`\`\`
`
} else {
userMessage += `## Initial App State
No initial app was provided - this is a new app created from scratch.
`
}
userMessage += `## Generated App
\`\`\`json
${JSON.stringify(generatedApp, null, 2)}
\`\`\`
Please evaluate how well the generated app:
1. Fulfills ALL requirements from the user's original request
2. ${initialApp ? 'Makes appropriate modifications to the initial app state' : 'Implements a complete and correct new app'}`
try {
const response = await client.messages.create({
model: 'claude-sonnet-4-6',
max_tokens: 2048,
system: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT,
messages: [
{ role: 'user', content: userMessage }
],
temperature: 0
})
const textBlock = response.content.find((block) => block.type === 'text')
const content = textBlock?.text
if (!content) {
return {
success: false,
resemblanceScore: 0,
statement: 'No response from evaluator',
error: 'Empty response from LLM'
}
}
// Parse JSON response - handle potential markdown code blocks
let jsonContent = content.trim()
if (jsonContent.startsWith('```')) {
jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '')
}
const parsed = JSON.parse(jsonContent) as {
resemblanceScore: number
statement: string
missingRequirements?: string[]
}
return {
success: true,
resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
statement: parsed.statement,
missingRequirements: parsed.missingRequirements ?? []
}
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err)
return {
success: false,
resemblanceScore: 0,
statement: 'Evaluation failed',
error: errorMessage
}
}
}

79
ai_evals/adapters/frontend/core/app/appEvalRunner.ts
View File

@@ -11,60 +11,29 @@ import {
prepareAppSystemMessage,
prepareAppUserMessage
} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
import { createAppFileHelpers } from './fileHelpers'
import { evaluateAppGeneration, type InitialApp } from './appEvalComparison'
import {
runEval,
resolveSystemPrompt,
resolveTools,
resolveModel,
type VariantConfig,
type BaseEvalResult,
type EvaluationResult,
type Tool,
type VariantDefaults
} from '../shared'
import { runEval } from '../shared'
import type { AIProvider } from '$lib/gen/types.gen'
// Re-export for convenience
export type { InitialApp } from './appEvalComparison'
/**
* App-specific evaluation result.
*/
export interface AppEvalResult extends BaseEvalResult<AppFiles> {
/** Alias for output to maintain API compatibility */
export interface AppEvalResult {
success: boolean
files: AppFiles
error?: string
assistantMessageCount: number
toolCallCount: number
toolsUsed: string[]
}
/**
* Options for running an app evaluation.
*/
export interface AppEvalOptions {
initialFrontend?: Record<string, string>
initialBackend?: Record<string, BackendRunnable>
model?: string
customSystemPrompt?: string
maxIterations?: number
variant?: VariantConfig
/** Whether to evaluate the generated app with LLM. Default: true. Set to false to skip evaluation. */
evaluateWithLLM?: boolean
/** AI provider (inferred from model name if omitted) */
provider?: AIProvider
workspaceRoot?: string
}
/**
* App-specific variant defaults.
*/
const appDefaults: VariantDefaults<AppAIChatHelpers> = {
prepareSystemMessage: prepareAppSystemMessage,
tools: getAppTools() as Tool<AppAIChatHelpers>[]
}
/**
* Runs an app chat evaluation using the shared chat loop (same code path as production).
*/
export async function runAppEval(
userPrompt: string,
apiKey: string,
@@ -80,14 +49,9 @@ export async function runAppEval(
)
try {
const variantName = options?.variant?.name ?? 'baseline'
const systemMessage = resolveSystemPrompt(
options?.variant,
appDefaults,
options?.customSystemPrompt
)
const { tools } = resolveTools(options?.variant, appDefaults)
const model = resolveModel(options?.variant, options?.model)
const systemMessage = prepareAppSystemMessage()
const tools = getAppTools() as ProductionTool<AppAIChatHelpers>[]
const model = options?.model ?? 'claude-haiku-4-5-20251001'
const userMessage = prepareAppUserMessage(userPrompt, helpers.getSelectedContext())
const rawResult = await runEval({
@@ -106,24 +70,13 @@ export async function runAppEval(
}
})
let evaluationResult: EvaluationResult | undefined
if (options?.evaluateWithLLM !== false) {
const generatedApp = getFiles()
const initialApp: InitialApp | undefined =
options?.initialFrontend || options?.initialBackend
? {
frontend: options.initialFrontend ?? {},
backend: options.initialBackend ?? {}
}
: undefined
evaluationResult = await evaluateAppGeneration(userPrompt, generatedApp, initialApp)
}
return {
...rawResult,
variantName,
files: rawResult.output,
evaluationResult
success: rawResult.success,
error: rawResult.error,
assistantMessageCount: rawResult.iterations,
toolCallCount: rawResult.toolCallsCount,
toolsUsed: rawResult.toolsCalled
}
} finally {
await cleanup()

89
ai_evals/adapters/frontend/core/evalCaseLoader.ts
View File

@@ -1,89 +0,0 @@
import type { ScriptLang } from "$lib/gen/types.gen";
import {
loadEvalCases,
type EvalScriptFixture
} from "../../shared/evalCases";
export interface FlowEvalCaseManifest {
id: string;
title: string;
userPrompt: string;
minJudgeScore?: number;
}
export interface FlowEvalCase extends FlowEvalCaseManifest {
expectedFlow: Record<string, unknown>;
initialFlow?: Record<string, any>;
}
export interface AppEvalCaseManifest {
id: string;
title: string;
userPrompt: string;
initialAppFixturePath?: string;
minJudgeScore?: number;
}
export interface AppEvalCase extends AppEvalCaseManifest {}
export interface ScriptEvalFixture {
code: string;
lang: ScriptLang | "bunnative";
path: string;
args: Record<string, any>;
}
export interface ScriptEvalCaseManifest {
id: string;
title: string;
userPrompt: string;
minJudgeScore?: number;
}
export interface ScriptEvalCase extends ScriptEvalCaseManifest {
expectedScript: ScriptEvalFixture;
initialScript?: ScriptEvalFixture;
}
export function loadFlowEvalCases(): FlowEvalCase[] {
return loadEvalCases("frontend-flow").map((testCase) => ({
id: testCase.id,
title: testCase.title,
userPrompt: testCase.userPrompt,
minJudgeScore: testCase.judgeRubric.minScore,
expectedFlow: testCase.artifactChecks.expectedFlow,
initialFlow: testCase.initialState.initialFlow as Record<string, any> | undefined
}));
}
export function loadAppEvalCases(): AppEvalCase[] {
return loadEvalCases("frontend-app").map((testCase) => ({
id: testCase.id,
title: testCase.title,
userPrompt: testCase.userPrompt,
initialAppFixturePath: testCase.initialState.initialAppFixturePath,
minJudgeScore: testCase.judgeRubric.minScore
}));
}
export function loadScriptEvalCases(): ScriptEvalCase[] {
return loadEvalCases("frontend-script").map((testCase) => ({
id: testCase.id,
title: testCase.title,
userPrompt: testCase.userPrompt,
minJudgeScore: testCase.judgeRubric.minScore,
expectedScript: toScriptEvalFixture(testCase.artifactChecks.expectedScript),
initialScript: testCase.initialState.initialScript
? toScriptEvalFixture(testCase.initialState.initialScript)
: undefined
}));
}
function toScriptEvalFixture(fixture: EvalScriptFixture): ScriptEvalFixture {
return {
code: fixture.code,
lang: fixture.lang as ScriptLang | "bunnative",
path: fixture.path,
args: (fixture.args as Record<string, any> | undefined) ?? {}
};
}

68
ai_evals/adapters/frontend/core/flow/flowEvalComparison.ts
View File

@@ -1,68 +0,0 @@
import type { FlowModule } from '$lib/gen'
import { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared'
import type { EvaluationResult } from '../shared'
/**
* Expected flow structure for evaluation.
*/
export interface ExpectedFlow {
summary?: string
value: {
modules: FlowModule[]
}
schema?: Record<string, any>
}
/**
* Flow-specific evaluator system prompt.
*/
const FLOW_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill flow definitions. Your task is to evaluate a generated flow against:
1. The original user request/prompt
2. An expected reference flow
## Windmill Flow Context
- Flows consist of modules (steps) that execute sequentially
- Module types include: rawscript, forloopflow, branchone, branchall, script, flow, aiagent
- Each module has an id, value (containing type and config), and may have input_transforms
- input_transforms connect modules using expressions like "results.previous_step". Valid input_transforms are: static, javascript. Valid variables in javascript expressions are: results, flow_input, flow_input.iter.value (for forloopflow), flow_input.iter.index (for forloopflow).
- forloopflow contains nested modules that execute per iteration with access to flow_input.iter.value
- branchone executes first matching branch, branchall executes all matching branches
- Branches have conditional expressions (expr) that determine execution
- aiagent modules contain tools array with tool definitions
## Evaluation Criteria
1. **User Request Fulfillment**: Does the generated flow address ALL requirements from the user's original prompt?
- Are all requested steps present?
- Are the requested features implemented (loops, branches, specific logic)?
- Does the schema match what the user requested for inputs?
2. **Structure**: Are the module types and nesting structure appropriate for the task?
3. **Logic**: Does the flow accomplish the intended logical task?
4. **Connections**: Are input_transforms connecting data correctly between steps?
5. **Completeness**: Are all required steps present with no major omissions?
6. **Code Quality**: Is the code functionally correct (exact syntax doesn't need to match)?
## Important Notes
- Minor differences in variable names, code formatting, or exact wording are acceptable
- Focus on functional equivalence, not character-by-character matching
- The generated flow should achieve the same outcome as described in the user request
- Extra helper steps or slightly different approaches can still score high if they accomplish the goal
- If the user requested specific module types (like aiagent), verify they are used correctly
${BASE_EVALUATOR_RESPONSE_FORMAT}`
/**
* Evaluates how well a generated flow matches an expected flow and user request using an LLM.
* Returns a resemblance score (0-100), a qualitative statement, and any missing requirements.
*/
export async function evaluateFlowComparison(
generatedFlow: ExpectedFlow,
expectedFlow: ExpectedFlow,
userPrompt: string
): Promise<EvaluationResult> {
return evaluateWithLLM({
userPrompt,
generatedOutput: generatedFlow,
expectedOutput: expectedFlow,
evaluatorSystemPrompt: FLOW_EVALUATOR_SYSTEM_PROMPT
})
}

95
ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts
View File

@@ -10,58 +10,34 @@ import {
prepareFlowUserMessage,
type FlowAIChatHelpers
} from '../../../../../frontend/src/lib/components/copilot/chat/flow/core'
import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
import { createFlowFileHelpers } from './fileHelpers'
import { evaluateFlowComparison, type ExpectedFlow } from './flowEvalComparison'
import {
runEval,
resolveSystemPrompt,
resolveTools,
resolveModel,
type VariantConfig,
type BaseEvalResult,
type EvaluationResult,
type Tool,
type VariantDefaults
} from '../shared'
import { runEval } from '../shared'
// Re-export for convenience
export type { ExpectedFlow } from './flowEvalComparison'
/**
* Flow-specific evaluation result.
*/
export interface FlowEvalResult extends BaseEvalResult<ExtendedOpenFlow> {
/** Alias for output to maintain API compatibility */
flow: ExtendedOpenFlow
export interface FlowFixture {
value?: {
modules?: FlowModule[]
}
schema?: Record<string, unknown>
}
export interface FlowEvalResult {
success: boolean
flow: ExtendedOpenFlow
error?: string
assistantMessageCount: number
toolCallCount: number
toolsUsed: string[]
}
/**
* Options for running a flow evaluation.
*/
export interface FlowEvalOptions {
initialModules?: FlowModule[]
initialSchema?: Record<string, any>
initialFlow?: FlowFixture
model?: string
customSystemPrompt?: string
maxIterations?: number
variant?: VariantConfig
expectedFlow?: ExpectedFlow
/** AI provider (inferred from model name if omitted) */
provider?: AIProvider
workspaceRoot?: string
}
/**
* Flow-specific variant defaults.
*/
const flowDefaults: VariantDefaults<FlowAIChatHelpers> = {
prepareSystemMessage: prepareFlowSystemMessage,
tools: flowTools as Tool<FlowAIChatHelpers>[]
}
/**
* Runs a flow chat evaluation using the shared chat loop (same code path as production).
*/
export async function runFlowEval(
userPrompt: string,
apiKey: string,
@@ -71,20 +47,15 @@ export async function runFlowEval(
options?.workspaceRoot ??
(await mkdtemp(join(tmpdir(), 'wmill-frontend-flow-benchmark-')))
const { helpers, getFlow, cleanup } = await createFlowFileHelpers(
options?.initialModules ?? [],
options?.initialSchema,
options?.initialFlow?.value?.modules ?? [],
options?.initialFlow?.schema,
workspaceRoot
)
try {
const variantName = options?.variant?.name ?? 'baseline'
const systemMessage = resolveSystemPrompt(
options?.variant,
flowDefaults,
options?.customSystemPrompt
)
const { tools } = resolveTools(options?.variant, flowDefaults)
const model = resolveModel(options?.variant, options?.model)
const systemMessage = prepareFlowSystemMessage()
const tools = flowTools as ProductionTool<FlowAIChatHelpers>[]
const model = options?.model ?? 'claude-haiku-4-5-20251001'
const userMessage = prepareFlowUserMessage(userPrompt, helpers.getFlowAndSelectedId(), [])
const rawResult = await runEval({
@@ -103,25 +74,13 @@ export async function runFlowEval(
}
})
let evaluationResult: EvaluationResult | undefined
if (options?.expectedFlow) {
const generatedFlow = getFlow()
evaluationResult = await evaluateFlowComparison(
{
summary: generatedFlow.summary,
value: { modules: generatedFlow.value.modules },
schema: generatedFlow.schema
},
options.expectedFlow,
userPrompt
)
}
return {
...rawResult,
variantName,
flow: rawResult.output,
evaluationResult
success: rawResult.success,
error: rawResult.error,
assistantMessageCount: rawResult.iterations,
toolCallCount: rawResult.toolCallsCount,
toolsUsed: rawResult.toolsCalled
}
} finally {
await cleanup()

36
ai_evals/adapters/frontend/core/script/scriptEvalComparison.ts
View File

@@ -1,36 +0,0 @@
import { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared'
import type { EvaluationResult } from '../shared'
import type { ScriptEvalState } from './fileHelpers'
const SCRIPT_EVALUATOR_SYSTEM_PROMPT = `You are an expert evaluator for Windmill script generation.
Your task is to evaluate a generated script against:
1. The original user request/prompt
2. An expected reference script
## Windmill Script Context
- Scripts should usually export a \`main\` function unless the request clearly requires a \`preprocessor\`
- The generated output may contain extra imports, helper functions, or comments
- Minor differences in formatting, variable names, or extra safe guards are acceptable
- Focus on whether the code fulfills the requested behavior and has the correct overall structure
## Evaluation Criteria
1. **User Request Fulfillment**: Does the generated script address the requested behavior?
2. **Entrypoint Correctness**: Does it export the right main entrypoint and shape?
3. **Functional Equivalence**: Does it implement the same logic as the expected script, even if syntax differs?
4. **Windmill Fit**: Is the script appropriate for a Windmill script context?
${BASE_EVALUATOR_RESPONSE_FORMAT}`
export async function evaluateScriptComparison(
generatedScript: ScriptEvalState,
expectedScript: ScriptEvalState,
userPrompt: string
): Promise<EvaluationResult> {
return evaluateWithLLM({
userPrompt,
generatedOutput: generatedScript,
expectedOutput: expectedScript,
evaluatorSystemPrompt: SCRIPT_EVALUATOR_SYSTEM_PROMPT
})
}

73
ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts
View File

@@ -9,31 +9,23 @@ import {
prepareScriptUserMessage,
type ScriptChatHelpers
} from '../../../../../frontend/src/lib/components/copilot/chat/script/core'
import type { Tool as ProductionTool } from '../../../../../frontend/src/lib/components/copilot/chat/shared'
import { createScriptFileHelpers, type ScriptEvalState } from './fileHelpers'
import { evaluateScriptComparison } from './scriptEvalComparison'
import {
runEval,
resolveSystemPrompt,
resolveTools,
resolveModel,
type VariantConfig,
type BaseEvalResult,
type EvaluationResult,
type Tool,
type VariantDefaults
} from '../shared'
import { runEval } from '../shared'
export interface ScriptEvalResult extends BaseEvalResult<ScriptEvalState> {
export interface ScriptEvalResult {
success: boolean
script: ScriptEvalState
error?: string
assistantMessageCount: number
toolCallCount: number
toolsUsed: string[]
}
export interface ScriptEvalOptions {
initialScript: ScriptEvalState
model?: string
customSystemPrompt?: string
maxIterations?: number
variant?: VariantConfig
expectedScript?: ScriptEvalState
provider?: AIProvider
workspaceRoot?: string
}
@@ -64,31 +56,19 @@ export async function runScriptEval(
)
try {
const variantName = options.variant?.name ?? 'baseline'
const model = resolveModel(options.variant, options.model)
const model = options.model ?? 'claude-haiku-4-5-20251001'
const modelProvider = resolveModelProvider(model, options.provider)
const selectedContext: ContextElement[] = []
const scriptDefaults: VariantDefaults<ScriptChatHelpers> = {
prepareSystemMessage: (customPrompt?: string) =>
prepareScriptSystemMessage(
modelProvider,
options.initialScript.lang,
{},
customPrompt ?? options.customSystemPrompt
),
tools: prepareScriptTools(
modelProvider,
options.initialScript.lang,
selectedContext
) as Tool<ScriptChatHelpers>[]
}
const systemMessage = resolveSystemPrompt(
options.variant,
scriptDefaults,
options.customSystemPrompt
const systemMessage = prepareScriptSystemMessage(
modelProvider,
options.initialScript.lang,
{}
)
const { tools } = resolveTools(options.variant, scriptDefaults)
const tools = prepareScriptTools(
modelProvider,
options.initialScript.lang,
selectedContext
) as ProductionTool<ScriptChatHelpers>[]
const userMessage = prepareScriptUserMessage(userPrompt, selectedContext)
const rawResult = await runEval({
@@ -107,20 +87,13 @@ export async function runScriptEval(
}
})
let evaluationResult: EvaluationResult | undefined
if (options.expectedScript) {
evaluationResult = await evaluateScriptComparison(
getScript(),
options.expectedScript,
userPrompt
)
}
return {
...rawResult,
variantName,
script: rawResult.output,
evaluationResult
success: rawResult.success,
error: rawResult.error,
assistantMessageCount: rawResult.iterations,
toolCallCount: rawResult.toolCallsCount,
toolsUsed: rawResult.toolsCalled
}
} finally {
await cleanup()

26
ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts
View File

@@ -5,29 +5,13 @@ import type {
ChatCompletionSystemMessageParam
} from 'openai/resources/chat/completions.mjs'
import type { AIProvider, AIProviderModel } from '$lib/gen/types.gen'
import type { TokenUsage, ToolCallDetail, EvalRunnerOptions } from './types'
import type { Tool } from './baseVariants'
import type { TokenUsage, ToolCallDetail, EvalRunnerOptions, RawEvalResult } from './types'
import { runChatLoop, type ChatClients } from '../../../../../frontend/src/lib/components/copilot/chat/chatLoop'
import type {
Tool as ProductionTool,
ToolCallbacks
} from '../../../../../frontend/src/lib/components/copilot/chat/shared'
/**
* Result from a single eval run (before domain-specific evaluation).
*/
export interface RawEvalResult<TOutput> {
success: boolean
output: TOutput
error?: string
tokenUsage: TokenUsage
toolCallsCount: number
toolsCalled: string[]
toolCallDetails: ToolCallDetail[]
iterations: number
messages: ChatCompletionMessageParam[]
}
/**
* Parameters for running a base evaluation.
*/
@@ -41,7 +25,7 @@ export interface RunEvalParams<THelpers, TOutput> {
/** Tool definitions for the LLM API (unused — derived from tools) */
toolDefs?: unknown
/** Full tool implementations for execution */
tools: Tool<THelpers>[]
tools: ProductionTool<THelpers>[]
/** Domain-specific helpers for tool execution */
helpers: THelpers
/** API key for the provider */
@@ -60,12 +44,12 @@ function createEvalClients(provider: AIProvider, apiKey: string): ChatClients {
return {
openai: new OpenAI({ apiKey: 'unused' }),
anthropic: new Anthropic({ apiKey })
}
} as ChatClients
}
return {
openai: new OpenAI({ apiKey }),
anthropic: new Anthropic({ apiKey: 'unused' })
}
} as ChatClients
}
/**
@@ -131,7 +115,7 @@ export async function runEval<THelpers, TOutput>(
}
return tool.fn(p)
}
})) as ProductionTool<THelpers>[]
}))
// No-op callbacks for eval
const callbacks: ToolCallbacks & {

135
ai_evals/adapters/frontend/core/shared/baseLLMEvaluator.ts
View File

@@ -1,135 +0,0 @@
import Anthropic from '@anthropic-ai/sdk'
import type { EvaluationResult } from './types'
/**
* Parameters for LLM-based evaluation.
*/
export interface EvaluateParams {
/** The user's original request/prompt */
userPrompt: string
/** The generated output to evaluate */
generatedOutput: unknown
/** The expected/reference output */
expectedOutput: unknown
/** Domain-specific system prompt for the evaluator */
evaluatorSystemPrompt: string
/** Anthropic API key for evaluation */
apiKey?: string
/** Model to use for evaluation (default: 'claude-sonnet-4-5-20250514') */
model?: string
}
/**
* Base evaluator system prompt template.
* Domain-specific evaluators should build on this structure.
*/
export const BASE_EVALUATOR_RESPONSE_FORMAT = `
## Response Format
You MUST respond with valid JSON only, no additional text:
{
"resemblanceScore": <0-100 integer>,
"statement": "<brief 1-2 sentence summary of how well the output matches the user request and expected output>",
"missingRequirements": ["<list any requirements from user prompt that are missing or incorrectly implemented>"]
}
Score guidelines:
- 90-100: Fully addresses user request, functionally equivalent to expected output
- 70-89: Addresses most user requirements, same overall structure with minor differences
- 50-69: Partially addresses user request, achieves similar goal but different approach
- 30-49: Missing significant requirements from user request
- 0-29: Does not address user request or significantly incorrect`
/**
* Evaluates how well a generated output matches an expected output using an LLM.
* Uses Anthropic API directly instead of OpenRouter.
*/
export async function evaluateWithLLM(params: EvaluateParams): Promise<EvaluationResult> {
const {
userPrompt,
generatedOutput,
expectedOutput,
evaluatorSystemPrompt,
apiKey,
model = 'claude-sonnet-4-6'
} = params
// @ts-ignore - process.env
const anthropicKey = apiKey ?? process.env.ANTHROPIC_API_KEY
if (!anthropicKey) {
return {
success: false,
resemblanceScore: 0,
statement: 'No API key available for evaluation',
error: 'ANTHROPIC_API_KEY not set and no apiKey provided'
}
}
const client = new Anthropic({ apiKey: anthropicKey })
const userMessage = `## User's Original Request
${userPrompt}
## Expected Reference Output
\`\`\`json
${JSON.stringify(expectedOutput, null, 2)}
\`\`\`
## Generated Output
\`\`\`json
${JSON.stringify(generatedOutput, null, 2)}
\`\`\`
Please evaluate how well the generated output:
1. Fulfills ALL requirements from the user's original request
2. Matches the structure and logic of the expected reference output`
try {
const response = await client.messages.create({
model,
max_tokens: 2048,
system: evaluatorSystemPrompt,
messages: [
{ role: 'user', content: userMessage }
],
temperature: 0
})
const textBlock = response.content.find((block) => block.type === 'text')
const content = textBlock?.text
if (!content) {
return {
success: false,
resemblanceScore: 0,
statement: 'No response from evaluator',
error: 'Empty response from LLM'
}
}
// Parse JSON response - handle potential markdown code blocks
let jsonContent = content.trim()
if (jsonContent.startsWith('```')) {
jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '')
}
const parsed = JSON.parse(jsonContent) as {
resemblanceScore: number
statement: string
missingRequirements?: string[]
}
return {
success: true,
resemblanceScore: Math.max(0, Math.min(100, Math.round(parsed.resemblanceScore))),
statement: parsed.statement,
missingRequirements: parsed.missingRequirements ?? []
}
} catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err)
return {
success: false,
resemblanceScore: 0,
statement: 'Evaluation failed',
error: errorMessage
}
}
}

108
ai_evals/adapters/frontend/core/shared/baseVariants.ts
View File

@@ -1,108 +0,0 @@
import type {
ChatCompletionFunctionTool,
ChatCompletionSystemMessageParam
} from 'openai/resources/chat/completions.mjs'
import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs'
import type { VariantConfig } from './types'
/**
* Generic tool interface that matches the structure used across chat modules.
*/
export interface Tool<THelpers> {
def: ChatCompletionFunctionTool
fn: (params: {
args: Record<string, unknown>
workspace: string
helpers: THelpers
toolCallbacks: {
setToolStatus: (...args: unknown[]) => void
removeToolStatus: (...args: unknown[]) => void
}
toolId: string
}) => Promise<string>
}
/**
* Domain-specific defaults for variant resolution.
*/
export interface VariantDefaults<THelpers> {
/** Function to prepare system message, optionally with custom prompt */
prepareSystemMessage: (customPrompt?: string) => ChatCompletionSystemMessageParam
/** Available tools for the domain */
tools: Tool<THelpers>[]
}
/**
* Resolves system prompt from variant config.
* Returns the appropriate ChatCompletionSystemMessageParam based on config.
*/
export function resolveSystemPrompt<THelpers>(
variant: VariantConfig | undefined,
defaults: VariantDefaults<THelpers>,
fallbackCustomPrompt?: string
): ChatCompletionSystemMessageParam {
if (!variant?.systemPrompt || variant.systemPrompt.type === 'default') {
return defaults.prepareSystemMessage(fallbackCustomPrompt)
}
if (variant.systemPrompt.type === 'default-with-custom') {
return defaults.prepareSystemMessage(variant.systemPrompt.custom)
}
// type === 'custom'
return {
role: 'system',
content: variant.systemPrompt.content
}
}
/**
* Resolves tools from variant config.
* Returns both the tool definitions (for API) and full tools (for execution).
*/
export function resolveTools<THelpers>(
variant: VariantConfig | undefined,
defaults: VariantDefaults<THelpers>
): {
toolDefs: ChatCompletionTool[]
tools: Tool<THelpers>[]
} {
if (!variant?.tools || variant.tools.type === 'default') {
return {
toolDefs: defaults.tools.map((t) => t.def),
tools: defaults.tools
}
}
if (variant.tools.type === 'subset') {
const includeList = (variant.tools as { type: 'subset'; include: string[] }).include
const subset = defaults.tools.filter((t) => includeList.includes(t.def.function.name))
return {
toolDefs: subset.map((t) => t.def),
tools: subset
}
}
if (variant.tools.type === 'custom') {
// Custom tools are typed as unknown[] in base VariantConfig but domain-specific
// code should ensure they are the correct Tool<THelpers> type
const customTools = variant.tools.tools as Tool<THelpers>[]
return {
toolDefs: customTools.map((t) => t.def),
tools: customTools
}
}
// Default fallback
return {
toolDefs: defaults.tools.map((t) => t.def),
tools: defaults.tools
}
}
/**
* Resolves model from variant config with fallback.
*/
export function resolveModel(variant?: VariantConfig, fallback?: string): string {
return variant?.model ?? fallback ?? 'gpt-4o'
}

19
ai_evals/adapters/frontend/core/shared/index.ts
View File

@@ -1,18 +1,3 @@
export type {
TokenUsage,
ToolCallDetail,
EvaluationResult,
BaseEvalResult,
VariantConfig,
EvalRunnerOptions,
ToolCallbacks
} from './types'
export type { Tool, VariantDefaults } from './baseVariants'
export { resolveSystemPrompt, resolveTools, resolveModel } from './baseVariants'
export type { RawEvalResult, RunEvalParams } from './baseEvalRunner'
export type { TokenUsage, ToolCallDetail, EvalRunnerOptions, RawEvalResult } from './types'
export type { RunEvalParams } from './baseEvalRunner'
export { runEval } from './baseEvalRunner'
export type { EvaluateParams } from './baseLLMEvaluator'
export { evaluateWithLLM, BASE_EVALUATOR_RESPONSE_FORMAT } from './baseLLMEvaluator'

87
ai_evals/adapters/frontend/core/shared/types.ts
View File

@@ -1,39 +1,25 @@
import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs'
import type { AIProvider } from '$lib/gen/types.gen'
/**
* Token usage tracking for LLM calls.
*/
export interface TokenUsage {
prompt: number
completion: number
total: number
}
/**
* Details of a single tool call made during evaluation.
*/
export interface ToolCallDetail {
name: string
arguments: Record<string, unknown>
}
/**
* Result of LLM-based comparison/evaluation.
*/
export interface EvaluationResult {
success: boolean
resemblanceScore: number
statement: string
missingRequirements?: string[]
error?: string
export interface EvalRunnerOptions {
maxIterations?: number
model?: string
workspace?: string
provider?: AIProvider
}
/**
* Base evaluation result that can be extended for domain-specific outputs.
* @template TOutput The domain-specific output type (e.g., flow definition, app files)
*/
export interface BaseEvalResult<TOutput> {
export interface RawEvalResult<TOutput> {
success: boolean
output: TOutput
error?: string
@@ -42,66 +28,5 @@ export interface BaseEvalResult<TOutput> {
toolsCalled: string[]
toolCallDetails: ToolCallDetail[]
iterations: number
variantName: string
evaluationResult?: EvaluationResult
messages: ChatCompletionMessageParam[]
}
/**
* Base configuration for a variant in eval testing.
* Allows customizing system prompt, tools, and model for comparison.
*
* Note: Domain-specific variants may extend this with custom tool configurations.
* See flow/flowEvalVariants.ts for an example with custom tools.
*/
export interface VariantConfig {
name: string
description?: string
/** System prompt configuration */
systemPrompt?:
| { type: 'default' }
| { type: 'default-with-custom'; custom: string }
| { type: 'custom'; content: string }
/** Tools configuration - basic types supported by shared code */
tools?:
| { type: 'default' }
| { type: 'subset'; include: string[] }
| { type: 'custom'; tools: unknown[] }
/** Model to use (default: 'gpt-4o') */
model?: string
}
/**
* Options for running an evaluation.
*/
export interface EvalRunnerOptions {
/** Maximum iterations for tool call loop (default: 20) */
maxIterations?: number
/** Model to use for LLM calls */
model?: string
/** Workspace ID for tool calls */
workspace?: string
/** AI provider (inferred from model name if omitted) */
provider?: AIProvider
}
/**
* No-op tool callbacks for eval testing.
*/
export interface ToolCallbacks {
setToolStatus: (id: string, status: { content?: string; result?: string; error?: string }) => void
removeToolStatus: (id: string) => void
}
/**
* Creates no-op tool callbacks for eval testing.
*/
export function createNoOpToolCallbacks(): ToolCallbacks {
return {
setToolStatus: () => {},
removeToolStatus: () => {}
}
}

101
ai_evals/adapters/frontend/progress.ts Normal file
View File

@@ -0,0 +1,101 @@
export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script'
export type FrontendBenchmarkProgressEvent =
| {
type: 'run-start'
surface: FrontendBenchmarkProgressSurface
totalCases: number
runs: number
concurrency: number
}
| {
type: 'attempt-start'
surface: FrontendBenchmarkProgressSurface
caseId: string
caseNumber: number
totalCases: number
attempt: number
runs: number
}
| {
type: 'attempt-finish'
surface: FrontendBenchmarkProgressSurface
caseId: string
caseNumber: number
totalCases: number
attempt: number
runs: number
passed: boolean
durationMs: number
judgeScore: number | null
error: string | null
}
export const FRONTEND_BENCHMARK_PROGRESS_PREFIX = 'WMILL_FRONTEND_AI_EVAL_PROGRESS '
export function emitFrontendBenchmarkProgress(event: FrontendBenchmarkProgressEvent): void {
process.stderr.write(
`${FRONTEND_BENCHMARK_PROGRESS_PREFIX}${JSON.stringify(event)}\n`
)
}
export function parseFrontendBenchmarkProgressLine(
line: string
): FrontendBenchmarkProgressEvent | null {
if (!line.startsWith(FRONTEND_BENCHMARK_PROGRESS_PREFIX)) {
return null
}
try {
const parsed = JSON.parse(
line.slice(FRONTEND_BENCHMARK_PROGRESS_PREFIX.length)
) as FrontendBenchmarkProgressEvent
return parsed?.type ? parsed : null
} catch {
return null
}
}
export function formatFrontendBenchmarkProgressEvent(
event: FrontendBenchmarkProgressEvent
): string {
switch (event.type) {
case 'run-start':
return `Running ${event.surface}: ${event.totalCases} cases x ${event.runs} run${event.runs === 1 ? '' : 's'}, concurrency ${event.concurrency}`
case 'attempt-start':
return `${formatCasePrefix(event.caseNumber, event.totalCases)} ${event.caseId} attempt ${event.attempt}/${event.runs}...`
case 'attempt-finish': {
const parts = [
`${formatCasePrefix(event.caseNumber, event.totalCases)} ${event.caseId} attempt ${event.attempt}/${event.runs} ${event.passed ? 'pass' : 'fail'}`,
formatDuration(event.durationMs)
]
if (event.judgeScore !== null) {
parts.push(`judge ${formatNumber(event.judgeScore)}`)
}
if (event.error) {
parts.push(truncateSingleLine(event.error, 120))
}
return parts.join(' | ')
}
}
}
function formatCasePrefix(caseNumber: number, totalCases: number): string {
return `[${caseNumber}/${totalCases}]`
}
function formatDuration(durationMs: number): string {
return `${formatNumber(durationMs / 1000)}s`
}
function formatNumber(value: number): string {
return Number.isInteger(value) ? String(value) : value.toFixed(1)
}
function truncateSingleLine(value: string, maxLength: number): string {
const normalized = value.replace(/\s+/g, ' ').trim()
if (normalized.length <= maxLength) {
return normalized
}
return `${normalized.slice(0, Math.max(0, maxLength - 3))}...`
}

193
ai_evals/adapters/frontend/runtime.ts
View File

@@ -1,65 +1,30 @@
import { execFile as execFileCallback } from 'node:child_process'
import { spawn } from 'node:child_process'
import { mkdtemp, readFile, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { promisify } from 'node:util'
import { fileURLToPath } from 'node:url'
import {
formatFrontendBenchmarkProgressEvent,
parseFrontendBenchmarkProgressLine
} from './progress'
import type { BenchmarkRunResult } from '../../core/types'
const execFile = promisify(execFileCallback)
const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url))
const FRONTEND_DIR = path.join(REPO_ROOT, 'frontend')
const FRONTEND_BENCHMARK_TEST = '../ai_evals/adapters/frontend/vitestAdapter.test.ts'
export type FrontendSurfaceName = 'frontend-flow' | 'frontend-app' | 'frontend-script'
export interface FrontendBenchmarkConfig {
provider?: 'anthropic' | 'openai'
model?: string
systemPrompt?: {
mode: 'append' | 'replace'
content: string
}
}
export interface FrontendAdapterAttempt {
attempt: number
passed: boolean
durationMs: number
assistantMessageCount: number
toolCallCount: number
toolsUsed: string[]
checks: Array<{ name: string; passed: boolean; required?: boolean }>
requiredFailedChecks: string[]
judgeScore: number | null
judgeStatement: string | null
error: string | null
}
export interface FrontendAdapterCaseResult {
caseId: string
attempts: FrontendAdapterAttempt[]
}
export interface FrontendAdapterPayload {
surface: 'flow' | 'app' | 'script'
runs: number
provider: string
model: string
judgeModel: string | null
caseResults: FrontendAdapterCaseResult[]
}
export type FrontendMode = 'flow' | 'app' | 'script'
export async function runFrontendBenchmarkAdapter(input: {
surface: FrontendSurfaceName
mode: FrontendMode
caseIds: string[]
runs: number
config?: FrontendBenchmarkConfig
}): Promise<FrontendAdapterPayload> {
}): Promise<BenchmarkRunResult> {
const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-'))
const outputPath = path.join(tempDir, 'result.json')
try {
await execFile(
await runVitestBenchmark(
path.join(FRONTEND_DIR, 'node_modules', '.bin', 'vitest'),
[
'run',
@@ -68,42 +33,124 @@ export async function runFrontendBenchmarkAdapter(input: {
'server',
'--config',
'vite.config.js'
],
{
cwd: FRONTEND_DIR,
env: {
...process.env,
WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath,
WMILL_FRONTEND_AI_EVAL_SURFACE: frontendSurfaceToAdapterSurface(input.surface),
WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds),
WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs),
WMILL_FRONTEND_AI_EVAL_CONFIG: input.config
? JSON.stringify(input.config)
: ''
},
maxBuffer: 10 * 1024 * 1024
],
{
cwd: FRONTEND_DIR,
env: {
...process.env,
BROWSERSLIST_IGNORE_OLD_DATA: '1',
WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH: outputPath,
WMILL_FRONTEND_AI_EVAL_MODE: input.mode,
WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds),
WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs),
WMILL_FRONTEND_AI_EVAL_PROGRESS: '1'
}
)
}
)
const raw = await readFile(outputPath, 'utf8')
return JSON.parse(raw) as FrontendAdapterPayload
return JSON.parse(raw) as BenchmarkRunResult
} catch (error) {
const executionError = error as Error & { stdout?: string; stderr?: string }
const details = [executionError.message, executionError.stdout, executionError.stderr]
.filter(Boolean)
.join('\n')
throw new Error(`Frontend benchmark adapter failed:\n${details}`)
throw new Error(`Frontend benchmark adapter failed:\n${toErrorMessage(error)}`)
} finally {
await rm(tempDir, { recursive: true, force: true })
}
}
function frontendSurfaceToAdapterSurface(surface: FrontendSurfaceName): 'flow' | 'app' | 'script' {
if (surface === 'frontend-flow') {
return 'flow'
async function runVitestBenchmark(
command: string,
args: string[],
options: {
cwd: string
env: NodeJS.ProcessEnv
}
if (surface === 'frontend-app') {
return 'app'
}
return 'script'
): Promise<void> {
const child = spawn(command, args, {
cwd: options.cwd,
env: options.env,
stdio: ['ignore', 'pipe', 'pipe']
})
let stdout = ''
let stderr = ''
let stderrLineBuffer = ''
child.stdout?.setEncoding('utf8')
child.stdout?.on('data', (chunk: string) => {
stdout += chunk
})
child.stderr?.setEncoding('utf8')
child.stderr?.on('data', (chunk: string) => {
stderrLineBuffer += chunk
const { remainder, passthrough } = drainProgressLines(stderrLineBuffer)
stderrLineBuffer = remainder
stderr += passthrough
})
await new Promise<void>((resolve, reject) => {
child.once('error', reject)
child.once('close', (code) => {
if (stderrLineBuffer.length > 0) {
const { remainder, passthrough } = drainProgressLines(`${stderrLineBuffer}\n`)
stderrLineBuffer = remainder
stderr += passthrough
}
if (code === 0) {
resolve()
return
}
const details = [`vitest exited with code ${code}`, stdout, stderr].filter(Boolean).join('\n')
reject(new Error(details))
})
})
}
function drainProgressLines(buffer: string): {
remainder: string
passthrough: string
} {
let remainder = buffer
let passthrough = ''
while (true) {
const newlineIndex = remainder.indexOf('\n')
if (newlineIndex === -1) {
return { remainder, passthrough }
}
const line = remainder.slice(0, newlineIndex).replace(/\r$/, '')
remainder = remainder.slice(newlineIndex + 1)
const progressEvent = parseFrontendBenchmarkProgressLine(line)
if (progressEvent) {
process.stderr.write(`${formatFrontendBenchmarkProgressEvent(progressEvent)}\n`)
continue
}
if (shouldSuppressFrontendStderrLine(line)) {
continue
}
passthrough += `${line}\n`
process.stderr.write(`${line}\n`)
}
}
function shouldSuppressFrontendStderrLine(line: string): boolean {
return (
line.startsWith('[baseline-browser-mapping] ') ||
line.startsWith('Browserslist: browsers data (caniuse-lite) is ') ||
line.includes('update-browserslist-db@latest') ||
line.includes('update-db#readme')
)
}
function toErrorMessage(error: unknown): string {
if (error instanceof Error) {
return error.message
}
return String(error)
}

18
ai_evals/adapters/frontend/vitestAdapter.test.ts
View File

@@ -33,16 +33,16 @@ vi.mock('$lib/components/vscode', () => ({}))
const benchmarkOutputPath = process.env.WMILL_FRONTEND_AI_EVAL_OUTPUT_PATH
const benchmarkIt = benchmarkOutputPath ? it : it.skip
benchmarkIt(
'runs the frontend benchmark adapter from environment input',
async () => {
const { runFrontendBenchmarkFromEnv } = await import('./benchmarkRunner')
const payload = await runFrontendBenchmarkFromEnv()
benchmarkIt(
'runs the frontend benchmark adapter from environment input',
async () => {
const { runFrontendBenchmarkFromEnv } = await import('./benchmarkRunner')
const payload = await runFrontendBenchmarkFromEnv()
const absoluteOutputPath = resolve(benchmarkOutputPath!)
await mkdir(dirname(absoluteOutputPath), { recursive: true })
await writeFile(absoluteOutputPath, JSON.stringify(payload, null, 2) + '\n', 'utf8')
expect(payload.caseResults.length).toBeGreaterThan(0)
},
600_000
)
expect(payload.cases.length).toBeGreaterThan(0)
},
600_000
)

344
ai_evals/adapters/shared/evalCases.ts
View File

@@ -1,344 +0,0 @@
import { readdirSync, readFileSync } from "node:fs";
import { join, resolve, dirname } from "node:path";
import { fileURLToPath } from "node:url";
export type EvalSurfaceName =
| "cli"
| "frontend-flow"
| "frontend-app"
| "frontend-script";
export interface EvalCaseSummary {
id: string;
surface: EvalSurfaceName;
title: string;
tags: string[];
}
export interface EvalJudgeRubric {
minScore?: number;
}
export interface CliExpectedFileCheck {
path: string;
mustContain?: string[];
mustNotContain?: string[];
}
interface RawCliExpectedFileCheck {
path: string;
must_contain?: string[];
must_not_contain?: string[];
}
export interface EvalScriptFixture {
code: string;
lang: string;
path: string;
args?: Record<string, unknown>;
}
interface ResolvedEvalCaseBase {
id: string;
surface: EvalSurfaceName;
title: string;
userPrompt: string;
workspaceContext: Record<string, unknown>;
judgeRubric: EvalJudgeRubric;
tags: string[];
}
export interface ResolvedCliEvalCase extends ResolvedEvalCaseBase {
surface: "cli";
initialState: Record<string, never>;
artifactChecks: {
expectedSkill?: string;
expectedOutputSubstrings: string[];
expectedFiles: CliExpectedFileCheck[];
};
}
export interface ResolvedFrontendFlowEvalCase extends ResolvedEvalCaseBase {
surface: "frontend-flow";
initialState: {
initialFlow?: Record<string, unknown>;
};
artifactChecks: {
expectedFlow: Record<string, unknown>;
};
}
export interface ResolvedFrontendAppEvalCase extends ResolvedEvalCaseBase {
surface: "frontend-app";
initialState: {
initialAppFixturePath?: string;
};
artifactChecks: Record<string, never>;
}
export interface ResolvedFrontendScriptEvalCase extends ResolvedEvalCaseBase {
surface: "frontend-script";
initialState: {
initialScript?: EvalScriptFixture;
};
artifactChecks: {
expectedScript: EvalScriptFixture;
};
}
export type ResolvedEvalCaseBySurface = {
cli: ResolvedCliEvalCase;
"frontend-flow": ResolvedFrontendFlowEvalCase;
"frontend-app": ResolvedFrontendAppEvalCase;
"frontend-script": ResolvedFrontendScriptEvalCase;
};
type RawJudgeRubric = {
min_score?: number;
};
type RawSharedEvalCase = {
id: string;
surface: EvalSurfaceName;
title: string;
user_prompt: string;
initial_state: Record<string, unknown>;
workspace_context: Record<string, unknown>;
artifact_checks: Record<string, unknown>;
judge_rubric: RawJudgeRubric;
tags: string[];
};
type RawCliEvalCase = RawSharedEvalCase & {
surface: "cli";
artifact_checks: {
expected_skill?: string;
expected_output_substrings?: string[];
expected_files?: RawCliExpectedFileCheck[];
};
};
type RawFrontendFlowEvalCase = RawSharedEvalCase & {
surface: "frontend-flow";
initial_state: {
flow_path?: string;
};
artifact_checks: {
expected_flow_path: string;
};
};
type RawFrontendAppEvalCase = RawSharedEvalCase & {
surface: "frontend-app";
initial_state: {
app_fixture_path?: string;
};
artifact_checks: Record<string, never>;
};
type RawFrontendScriptEvalCase = RawSharedEvalCase & {
surface: "frontend-script";
initial_state: {
script_path?: string;
};
artifact_checks: {
expected_script_path: string;
};
};
type RawEvalCaseBySurface = {
cli: RawCliEvalCase;
"frontend-flow": RawFrontendFlowEvalCase;
"frontend-app": RawFrontendAppEvalCase;
"frontend-script": RawFrontendScriptEvalCase;
};
const REPO_ROOT = resolve(
dirname(fileURLToPath(import.meta.url)),
"../../.."
);
export function loadEvalCaseSummaries(surface: EvalSurfaceName): EvalCaseSummary[] {
return loadRawEvalCases(surface).map((entry) => ({
id: entry.id,
surface: entry.surface,
title: entry.title,
tags: [...(entry.tags ?? [])]
}));
}
export function loadEvalCases<T extends EvalSurfaceName>(
surface: T
): Array<ResolvedEvalCaseBySurface[T]> {
return loadRawEvalCases(surface).map((entry) =>
resolveEvalCase(entry)
) as Array<ResolvedEvalCaseBySurface[T]>;
}
function loadRawEvalCases<T extends EvalSurfaceName>(
surface: T
): Array<RawEvalCaseBySurface[T]> {
const manifestPaths = getManifestPaths(surface);
const cases: Array<RawEvalCaseBySurface[T]> = [];
for (const manifestPath of manifestPaths) {
const parsed = JSON.parse(readFileSync(manifestPath, "utf8")) as Array<
RawEvalCaseBySurface[T]
>;
if (!Array.isArray(parsed) || parsed.length === 0) {
throw new Error(`No eval cases found in ${manifestPath}`);
}
for (const entry of parsed) {
if (entry.surface !== surface) {
throw new Error(
`Eval case ${entry.id} in ${manifestPath} declared surface ${entry.surface}, expected ${surface}`
);
}
cases.push(entry);
}
}
return cases;
}
function resolveEvalCase(
entry:
| RawCliEvalCase
| RawFrontendFlowEvalCase
| RawFrontendAppEvalCase
| RawFrontendScriptEvalCase
):
| ResolvedCliEvalCase
| ResolvedFrontendFlowEvalCase
| ResolvedFrontendAppEvalCase
| ResolvedFrontendScriptEvalCase {
const base = {
id: entry.id,
surface: entry.surface,
title: entry.title,
userPrompt: entry.user_prompt,
workspaceContext: entry.workspace_context ?? {},
judgeRubric: normalizeJudgeRubric(entry.judge_rubric),
tags: [...(entry.tags ?? [])]
};
switch (entry.surface) {
case "cli":
return {
...base,
surface: "cli",
initialState: {},
artifactChecks: {
expectedSkill: entry.artifact_checks.expected_skill,
expectedOutputSubstrings:
entry.artifact_checks.expected_output_substrings ?? [],
expectedFiles: (entry.artifact_checks.expected_files ?? []).map(
(file) => ({
path: file.path,
mustContain: file.must_contain,
mustNotContain: file.must_not_contain
})
)
}
};
case "frontend-flow":
return {
...base,
surface: "frontend-flow",
initialState: {
initialFlow: entry.initial_state.flow_path
? readRepoRelativeJson<Record<string, unknown>>(
entry.initial_state.flow_path
)
: undefined
},
artifactChecks: {
expectedFlow: readRepoRelativeJson<Record<string, unknown>>(
entry.artifact_checks.expected_flow_path
)
}
};
case "frontend-app":
return {
...base,
surface: "frontend-app",
initialState: {
initialAppFixturePath: entry.initial_state.app_fixture_path
? resolveRepoRelativePath(entry.initial_state.app_fixture_path)
: undefined
},
artifactChecks: {}
};
case "frontend-script":
return {
...base,
surface: "frontend-script",
initialState: {
initialScript: entry.initial_state.script_path
? readRepoRelativeJson<EvalScriptFixture>(
entry.initial_state.script_path
)
: undefined
},
artifactChecks: {
expectedScript: readRepoRelativeJson<EvalScriptFixture>(
entry.artifact_checks.expected_script_path
)
}
};
default:
return assertNever(entry);
}
}
function getManifestPaths(surface: EvalSurfaceName): string[] {
if (surface === "cli") {
const cliCasesDir = join(REPO_ROOT, "ai_evals", "cases", "cli");
return readdirSync(cliCasesDir)
.filter((entry) => entry.endsWith(".json"))
.sort((left, right) => left.localeCompare(right))
.map((entry) => join(cliCasesDir, entry));
}
return [
join(
REPO_ROOT,
"ai_evals",
"cases",
"frontend",
`${surfaceToFrontendManifestName(surface)}.json`
)
];
}
function surfaceToFrontendManifestName(
surface: Exclude<EvalSurfaceName, "cli">
): "flow" | "app" | "script" {
if (surface === "frontend-flow") {
return "flow";
}
if (surface === "frontend-app") {
return "app";
}
return "script";
}
function normalizeJudgeRubric(value: RawJudgeRubric | undefined): EvalJudgeRubric {
return {
minScore: value?.min_score
};
}
function readRepoRelativeJson<T>(relativePath: string): T {
return JSON.parse(readFileSync(resolveRepoRelativePath(relativePath), "utf8")) as T;
}
function resolveRepoRelativePath(relativePath: string): string {
return join(REPO_ROOT, relativePath);
}
function assertNever(value: never): never {
throw new Error(`Unexpected value: ${JSON.stringify(value)}`);
}

473
ai_evals/adapters/shared/validators.ts
View File

@@ -1,473 +0,0 @@
import ts from "typescript";
export interface BenchmarkCheck {
name: string;
passed: boolean;
required?: boolean;
details?: string;
}
interface ScriptLikeArtifact {
code: string;
lang: string;
path: string;
}
interface FlowLikeArtifact {
value?: {
modules?: Array<Record<string, unknown>>;
};
schema?: Record<string, unknown>;
}
interface AppLikeFiles {
frontend: Record<string, string>;
backend: Record<string, AppLikeBackendRunnable>;
}
interface AppLikeBackendRunnable {
type?: string;
name?: string;
path?: string;
inlineScript?: {
language?: string;
content?: string;
};
}
interface CliExpectedFileCheck {
path: string;
mustContain?: string[];
mustNotContain?: string[];
}
interface CliFileArtifactResult {
path: string;
exists: boolean;
content?: string;
}
const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]);
export function requiredCheck(
name: string,
passed: boolean,
details?: string
): BenchmarkCheck {
return {
name,
passed,
required: true,
...(details ? { details } : {})
};
}
export function optionalCheck(
name: string,
passed: boolean,
details?: string
): BenchmarkCheck {
return {
name,
passed,
required: false,
...(details ? { details } : {})
};
}
export function allRequiredChecksPassed(checks: BenchmarkCheck[]): boolean {
return checks.every((check) => check.required === false || check.passed);
}
export function getRequiredFailedChecks(checks: BenchmarkCheck[]): string[] {
return checks
.filter((check) => check.required !== false && !check.passed)
.map((check) => check.name);
}
export function buildJudgeChecks(input: {
evaluationResult:
| {
success: boolean;
resemblanceScore: number;
error?: string;
}
| undefined;
minJudgeScore: number;
}): BenchmarkCheck[] {
return [
requiredCheck(
"judge evaluation succeeded",
Boolean(input.evaluationResult?.success),
input.evaluationResult?.error
),
requiredCheck(
`judge score >= ${input.minJudgeScore}`,
(input.evaluationResult?.resemblanceScore ?? 0) >= input.minJudgeScore,
`score=${input.evaluationResult?.resemblanceScore ?? 0}`
)
];
}
export function validateCliArtifact(input: {
assistantOutput: string;
skillsInvoked: string[];
expectedSkill?: string;
expectedOutputSubstrings?: string[];
expectedFiles: CliExpectedFileCheck[];
fileResults: CliFileArtifactResult[];
}): BenchmarkCheck[] {
const checks: BenchmarkCheck[] = [];
if (input.expectedSkill) {
checks.push(
requiredCheck(
`invokes ${input.expectedSkill}`,
input.skillsInvoked.includes(input.expectedSkill),
`skills invoked: ${input.skillsInvoked.join(", ")}`
)
);
}
for (const expectedOutput of input.expectedOutputSubstrings ?? []) {
checks.push(
requiredCheck(
`mentions '${expectedOutput}' in assistant output`,
input.assistantOutput.includes(expectedOutput)
)
);
}
for (const expectedFile of input.expectedFiles) {
const fileResult = input.fileResults.find((entry) => entry.path === expectedFile.path);
const content = fileResult?.content ?? "";
checks.push(
requiredCheck(`creates ${expectedFile.path}`, Boolean(fileResult?.exists))
);
for (const requiredSnippet of expectedFile.mustContain ?? []) {
checks.push(
requiredCheck(
`${expectedFile.path} contains '${requiredSnippet}'`,
content.includes(requiredSnippet)
)
);
}
for (const forbiddenSnippet of expectedFile.mustNotContain ?? []) {
checks.push(
requiredCheck(
`${expectedFile.path} avoids '${forbiddenSnippet}'`,
!content.includes(forbiddenSnippet)
)
);
}
}
return checks;
}
export function validateScriptArtifact(input: {
generatedScript: ScriptLikeArtifact;
expectedScript: ScriptLikeArtifact;
initialScript?: ScriptLikeArtifact;
}): BenchmarkCheck[] {
const lintErrors = getScriptLintErrors(input.generatedScript.code, input.generatedScript.lang);
const normalizedGenerated = normalizeText(input.generatedScript.code);
const normalizedInitial = input.initialScript
? normalizeText(input.initialScript.code)
: null;
return [
requiredCheck(
"script path matches expected",
input.generatedScript.path === input.expectedScript.path,
`expected ${input.expectedScript.path}, got ${input.generatedScript.path}`
),
requiredCheck(
"script language matches expected",
input.generatedScript.lang === input.expectedScript.lang,
`expected ${input.expectedScript.lang}, got ${input.generatedScript.lang}`
),
requiredCheck(
"script exports entrypoint",
hasSupportedEntrypoint(input.generatedScript.code)
),
requiredCheck(
"script has no syntax errors",
lintErrors.length === 0,
lintErrors.join(" | ")
),
...(normalizedInitial === null
? []
: [
requiredCheck(
"script differs from initial input",
normalizedGenerated !== normalizedInitial
)
])
];
}
export function validateFlowArtifact(input: {
generatedFlow: FlowLikeArtifact;
expectedFlow: FlowLikeArtifact;
}): BenchmarkCheck[] {
const generatedModules = getFlowModules(input.generatedFlow);
const expectedModules = getFlowModules(input.expectedFlow);
const generatedTypes = collectFlowModuleTypes(generatedModules);
const expectedTypes = collectFlowModuleTypes(expectedModules);
const missingTypes = [...expectedTypes].filter((type) => !generatedTypes.has(type));
const generatedTopLevelIds = getTopLevelFlowModuleIds(input.generatedFlow);
const expectedTopLevelIds = getTopLevelFlowModuleIds(input.expectedFlow);
const missingTopLevelIds = expectedTopLevelIds.filter(
(id) => !generatedTopLevelIds.includes(id)
);
const expectedSchemaType = getSchemaRootType(input.expectedFlow.schema);
const generatedSchemaType = getSchemaRootType(input.generatedFlow.schema);
return [
requiredCheck("flow has modules", generatedModules.length > 0),
requiredCheck(
"flow includes expected module types",
missingTypes.length === 0,
missingTypes.length > 0 ? `missing types: ${missingTypes.join(", ")}` : undefined
),
...(expectedSchemaType
? [
requiredCheck(
"flow schema root type matches expected",
generatedSchemaType === expectedSchemaType,
`expected ${expectedSchemaType}, got ${generatedSchemaType ?? "(missing)"}`
)
]
: []),
optionalCheck(
"flow includes expected top-level step ids",
missingTopLevelIds.length === 0,
missingTopLevelIds.length > 0
? `missing ids: ${missingTopLevelIds.join(", ")}`
: undefined
)
];
}
export function validateAppArtifact(input: {
generatedApp: AppLikeFiles;
initialApp?: AppLikeFiles;
}): BenchmarkCheck[] {
const frontendEntries = Object.entries(input.generatedApp.frontend ?? {});
const emptyFrontendFiles = frontendEntries
.filter(([, content]) => normalizeText(content).length === 0)
.map(([path]) => path);
const backendReferenceKeys = collectBackendReferences(
frontendEntries.map(([, content]) => content)
);
const missingBackendReferences = backendReferenceKeys.filter(
(key) => input.generatedApp.backend[key] === undefined
);
const invalidInlineRunnables = Object.entries(input.generatedApp.backend ?? {})
.filter(([, runnable]) => runnable.type === "inline")
.filter(([, runnable]) => !hasSupportedEntrypoint(runnable.inlineScript?.content ?? ""))
.map(([key]) => key);
const hasChangedFromInitial = input.initialApp
? !appFilesEqual(input.generatedApp, input.initialApp)
: true;
return [
requiredCheck("app has frontend files", frontendEntries.length > 0),
requiredCheck(
"app has frontend entrypoint",
Object.keys(input.generatedApp.frontend ?? {}).some(
(filePath) => filePath === "/index.tsx" || filePath === "/index.jsx"
)
),
requiredCheck(
"frontend files are non-empty",
emptyFrontendFiles.length === 0,
emptyFrontendFiles.length > 0
? `empty files: ${emptyFrontendFiles.join(", ")}`
: undefined
),
requiredCheck(
"frontend backend references resolve",
missingBackendReferences.length === 0,
missingBackendReferences.length > 0
? `missing runnables: ${missingBackendReferences.join(", ")}`
: undefined
),
requiredCheck(
"inline backend runnables export entrypoint",
invalidInlineRunnables.length === 0,
invalidInlineRunnables.length > 0
? `invalid inline runnables: ${invalidInlineRunnables.join(", ")}`
: undefined
),
...(input.initialApp
? [requiredCheck("app differs from initial input", hasChangedFromInitial)]
: [])
];
}
function hasSupportedEntrypoint(code: string): boolean {
return (
/export\s+(async\s+)?function\s+main\s*\(/.test(code) ||
/export\s+(async\s+)?function\s+preprocessor\s*\(/.test(code)
);
}
function getScriptLintErrors(code: string, lang: string): string[] {
if (!TS_LIKE_LANGUAGES.has(lang)) {
return hasSupportedEntrypoint(code)
? []
: ["Script must export a main or preprocessor function."];
}
const output = ts.transpileModule(code, {
compilerOptions: {
target: ts.ScriptTarget.ES2022,
module: ts.ModuleKind.ESNext,
moduleResolution: ts.ModuleResolutionKind.Bundler,
noEmit: true,
allowJs: true,
checkJs: false,
strict: false,
skipLibCheck: true
},
fileName: "script.ts",
reportDiagnostics: true
});
const diagnostics = (output.diagnostics ?? []).map((diagnostic) =>
ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")
);
if (!hasSupportedEntrypoint(code)) {
diagnostics.push("Script must export a main or preprocessor function.");
}
return diagnostics;
}
function getFlowModules(flow: FlowLikeArtifact): Array<Record<string, unknown>> {
const rootModules = Array.isArray(flow.value?.modules) ? flow.value.modules : [];
const collected: Array<Record<string, unknown>> = [];
for (const module of rootModules) {
visitFlowModule(module, collected);
}
return collected;
}
function visitFlowModule(
module: Record<string, unknown>,
collected: Array<Record<string, unknown>>
): void {
collected.push(module);
const value = asRecord(module.value);
const nestedModules = Array.isArray(value?.modules) ? value.modules : [];
for (const nested of nestedModules) {
if (isRecord(nested)) {
visitFlowModule(nested, collected);
}
}
const branches = Array.isArray(value?.branches) ? value.branches : [];
for (const branch of branches) {
const branchRecord = asRecord(branch);
const branchModules = Array.isArray(branchRecord?.modules)
? branchRecord.modules
: [];
for (const nested of branchModules) {
if (isRecord(nested)) {
visitFlowModule(nested, collected);
}
}
}
const defaultModules = Array.isArray(value?.default) ? value.default : [];
for (const nested of defaultModules) {
if (isRecord(nested)) {
visitFlowModule(nested, collected);
}
}
}
function collectFlowModuleTypes(
modules: Array<Record<string, unknown>>
): Set<string> {
const types = new Set<string>();
for (const module of modules) {
const value = asRecord(module.value);
if (typeof value?.type === "string") {
types.add(value.type);
}
}
return types;
}
function getTopLevelFlowModuleIds(flow: FlowLikeArtifact): string[] {
const rootModules = Array.isArray(flow.value?.modules) ? flow.value.modules : [];
return rootModules
.map((module) => (isRecord(module) && typeof module.id === "string" ? module.id : null))
.filter((id): id is string => id !== null);
}
function getSchemaRootType(schema: Record<string, unknown> | undefined): string | null {
return typeof schema?.type === "string" ? schema.type : null;
}
function collectBackendReferences(frontendContents: string[]): string[] {
const references = new Set<string>();
const backendCallPattern = /backend\.([A-Za-z0-9_]+)\s*\(/g;
for (const content of frontendContents) {
for (const match of content.matchAll(backendCallPattern)) {
const key = match[1];
if (key) {
references.add(key);
}
}
}
return [...references].sort((left, right) => left.localeCompare(right));
}
function appFilesEqual(left: AppLikeFiles, right: AppLikeFiles): boolean {
return stableStringify(left) === stableStringify(right);
}
function stableStringify(value: unknown): string {
return JSON.stringify(sortJsonValue(value));
}
function sortJsonValue(value: unknown): unknown {
if (Array.isArray(value)) {
return value.map(sortJsonValue);
}
if (!isRecord(value)) {
return value;
}
return Object.fromEntries(
Object.entries(value)
.sort((left, right) => left[0].localeCompare(right[0]))
.map(([key, nestedValue]) => [key, sortJsonValue(nestedValue)])
);
}
function normalizeText(value: string): string {
return value.replace(/\r\n/g, "\n").trim();
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
function asRecord(value: unknown): Record<string, unknown> | null {
return isRecord(value) ? value : null;
}

59
ai_evals/bun.lock
View File

@@ -6,6 +6,7 @@
"name": "windmill-ai-evals",
"dependencies": {
"@anthropic-ai/claude-agent-sdk": "^0.2.25",
"@anthropic-ai/sdk": "^0.39.0",
"commander": "^14.0.3",
},
"devDependencies": {
@@ -17,7 +18,7 @@
"packages": {
"@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.2.87", "", { "dependencies": { "@anthropic-ai/sdk": "^0.74.0", "@modelcontextprotocol/sdk": "^1.27.1" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.34.2", "@img/sharp-darwin-x64": "^0.34.2", "@img/sharp-linux-arm": "^0.34.2", "@img/sharp-linux-arm64": "^0.34.2", "@img/sharp-linux-x64": "^0.34.2", "@img/sharp-linuxmusl-arm64": "^0.34.2", "@img/sharp-linuxmusl-x64": "^0.34.2", "@img/sharp-win32-arm64": "^0.34.2", "@img/sharp-win32-x64": "^0.34.2" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-WWmgBPxPhBOvNT0ujI8vPTI2lK+w5YEkEZ/y1mH0EDkK/0kBnxVJNhCtG5vnueiAViwLoUOFn66pbkDiivijdA=="],
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.74.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw=="],
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.39.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg=="],
"@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="],
@@ -59,14 +60,22 @@
"@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
"@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
"@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
"@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="],
"abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
"accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
"agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="],
"ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="],
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
"body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
"bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
@@ -77,6 +86,8 @@
"call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="],
"combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="],
"commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="],
"content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="],
@@ -93,6 +104,8 @@
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
"delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
"depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
@@ -107,10 +120,14 @@
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
"es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="],
"escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
"etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="],
"event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="],
"eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="],
"eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
@@ -125,6 +142,12 @@
"finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="],
"form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="],
"form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="],
"formdata-node": ["formdata-node@4.4.1", "", { "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" } }, "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ=="],
"forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="],
"fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="],
@@ -139,12 +162,16 @@
"has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
"has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="],
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
"hono": ["hono@4.12.9", "", {}, "sha512-wy3T8Zm2bsEvxKZM5w21VdHDDcwVS1yUFFY6i8UobSsKfFceT7TOwhbhfKsDyx7tYQlmRM5FLpIuYvNFyjctiA=="],
"http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="],
"humanize-ms": ["humanize-ms@1.2.1", "", { "dependencies": { "ms": "^2.0.0" } }, "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ=="],
"iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
@@ -179,6 +206,10 @@
"negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],
"node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
"node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="],
"object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
"object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="],
@@ -231,18 +262,26 @@
"toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
"tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
"ts-algebra": ["ts-algebra@2.0.0", "", {}, "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw=="],
"type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
"undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
"unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="],
"vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="],
"web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="],
"webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
"whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
@@ -250,5 +289,19 @@
"zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
"zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="],
"@anthropic-ai/claude-agent-sdk/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.74.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw=="],
"@types/node-fetch/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
"bun-types/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
"form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
"@types/node-fetch/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
"bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
"form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="],
}
}

44
ai_evals/cases/app.json Normal file
View File

@@ -0,0 +1,44 @@
[
{
"id": "app-test1-counter-create",
"prompt": "Create a counter app with increment/decrement buttons"
},
{
"id": "app-test2-counter-reset",
"prompt": "Add a reset button that sets the counter back to 0",
"initial": "ai_evals/fixtures/frontend/app/initial/test1_counter_app"
},
{
"id": "app-test3-shopping-cart-quantity",
"prompt": "Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items",
"initial": "ai_evals/fixtures/frontend/app/initial/shopping_cart"
},
{
"id": "app-test4-shopping-cart-discount",
"prompt": "Add a discount code input field in the cart. When the code \"SAVE10\" is entered, apply a 10% discount to the total",
"initial": "ai_evals/fixtures/frontend/app/initial/shopping_cart"
},
{
"id": "app-test5-file-manager-search",
"prompt": "Add a search bar in the toolbar that filters files and folders by name as the user types",
"initial": "ai_evals/fixtures/frontend/app/initial/file_manager"
},
{
"id": "app-test6-file-manager-details",
"prompt": "Show file size (formatted as KB/MB) and modified date in the file list for each item",
"initial": "ai_evals/fixtures/frontend/app/initial/file_manager"
},
{
"id": "app-test7-file-manager-select-all",
"prompt": "Add a \"Select All\" checkbox in the file list header and individual checkboxes for each file. Add a \"Delete Selected\" button that appears when items are selected",
"initial": "ai_evals/fixtures/frontend/app/initial/file_manager"
},
{
"id": "app-test8-quiz-create",
"prompt": "Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct."
},
{
"id": "app-test9-recipe-book-create",
"prompt": "Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes."
}
]

12
ai_evals/cases/cli.json Normal file
View File

@@ -0,0 +1,12 @@
[
{
"id": "bun-hello-script",
"prompt": "This is a benchmark harness. Create exactly one Windmill Bun/TypeScript script at {{workspace_root}}/f/evals/hello.ts. The script must export async function main(name: string) and return an object { greeting: `Hello, ${name}!` }. Keep it minimal. Do not create other scripts. Do not run any CLI commands. After writing the file, tell me exactly which wmill commands I should run next.",
"expected": "ai_evals/fixtures/cli/expected/bun-hello-script"
},
{
"id": "bun-hello-flow",
"prompt": "This is a benchmark harness. Create exactly one Windmill flow folder at {{workspace_root}}/f/evals/hello__flow. The flow must contain flow.yaml and one inline Bun script file named hello.ts. The flow should accept a name string input and return an object { greeting: `Hello, ${name}!` }. Use a single rawscript step wired to that input. Keep it minimal. Do not create any other flows or scripts. Do not run any CLI commands. After writing the files, tell me exactly which wmill commands I should run next.",
"expected": "ai_evals/fixtures/cli/expected/bun-hello-flow"
}
]

42
ai_evals/cases/cli/flow.json
View File

@@ -1,42 +0,0 @@
[
{
"id": "bun-hello-flow",
"surface": "cli",
"title": "Create a minimal Bun flow in a fresh CLI workspace.",
"user_prompt": "This is a benchmark harness. Create exactly one Windmill flow folder at {{workspace_root}}/f/evals/hello__flow. The flow must contain flow.yaml and one inline Bun script file named hello.ts. The flow should accept a name string input and return an object { greeting: `Hello, ${name}!` }. Use a single rawscript step wired to that input. Keep it minimal. Do not create any other flows or scripts. Do not run any CLI commands. After writing the files, tell me exactly which wmill commands I should run next.",
"initial_state": {},
"workspace_context": {
"max_turns": 8
},
"artifact_checks": {
"expected_skill": "write-flow",
"expected_output_substrings": [
"wmill flow generate-locks",
"wmill sync push"
],
"expected_files": [
{
"path": "f/evals/hello__flow/flow.yaml",
"must_contain": [
"value:",
"modules:",
"name:"
]
},
{
"path": "f/evals/hello__flow/hello.ts",
"must_contain": [
"export async function main(name: string)",
"greeting: `Hello, ${name}!`"
]
}
]
},
"judge_rubric": {},
"tags": [
"cli",
"flow",
"create"
]
}
]

34
ai_evals/cases/cli/script.json
View File

@@ -1,34 +0,0 @@
[
{
"id": "bun-hello-script",
"surface": "cli",
"title": "Create a minimal Bun script in a fresh CLI workspace.",
"user_prompt": "This is a benchmark harness. Create exactly one Windmill Bun/TypeScript script at {{workspace_root}}/f/evals/hello.ts. The script must export async function main(name: string) and return an object { greeting: `Hello, ${name}!` }. Keep it minimal. Do not create other scripts. Do not run any CLI commands. After writing the file, tell me exactly which wmill commands I should run next.",
"initial_state": {},
"workspace_context": {
"max_turns": 6
},
"artifact_checks": {
"expected_skill": "write-script-bun",
"expected_output_substrings": [
"wmill script generate-metadata",
"wmill sync push"
],
"expected_files": [
{
"path": "f/evals/hello.ts",
"must_contain": [
"export async function main(name: string)",
"return { greeting: `Hello, ${name}!` };"
]
}
]
},
"judge_rubric": {},
"tags": [
"cli",
"script",
"create"
]
}
]

45
ai_evals/cases/flow.json Normal file
View File

@@ -0,0 +1,45 @@
[
{
"id": "flow-test0-sum-two-numbers",
"prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nCreate a flow with a single Bun rawscript step named \"sum_numbers\".\nThe flow input must be two numbers named a and b.\nThe rawscript must read a and b from flow input and return a + b.\nDo not add extra steps, branches, loops, AI agents, or test steps.",
"expected": "ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json"
},
{
"id": "flow-test1-user-role-actions",
"prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Fetch mock users from api\nSTEP 2: Filter only active users:\nSTEP 3: Loop on all users\nSTEP 4: Do branches based on user's role, do different action based on that. Roles are admin, user, moderator\nSTEP 5: Return action taken for each user",
"expected": "ai_evals/fixtures/frontend/flow/expected/test1.json"
},
{
"id": "flow-test2-order-processing",
"prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Receive order data from input (order has items array with name/price/quantity, customer_email, shipping_address)\nSTEP 2: Validate order - check all items have valid price > 0 and quantity > 0, return validation result\nSTEP 3: Calculate order total with 8% tax rate\nSTEP 4: Check inventory for each item (loop through items, return mock availability)\nSTEP 5: Branch based on inventory - if all items available, create shipment record; otherwise create backorder record\nSTEP 6: Send confirmation (mock email to customer_email)\nSTEP 7: Return final order summary with status",
"expected": "ai_evals/fixtures/frontend/flow/expected/test2.json"
},
{
"id": "flow-test3-data-pipeline",
"prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Fetch list of data sources from configuration (return mock array of 3 source objects with id and url)\nSTEP 2: For each data source in parallel:\n - Fetch raw data from the source (mock fetch returning sample records)\n - Transform/clean the data (filter out invalid entries)\n - Validate the transformed data (return validation score 0-100)\nSTEP 3: Aggregate all validated data into single dataset with combined records\nSTEP 4: Calculate overall data quality score (average of all validation scores)\nSTEP 5: Branch based on quality score:\n - If score >= 90: Store in primary database and return success\n - If score >= 70 and < 90: Store in secondary database with warning flag\n - If score < 70: Store in quarantine and send alert\nSTEP 6: Return processing report with statistics (total records, quality score, destination)",
"expected": "ai_evals/fixtures/frontend/flow/expected/test3.json"
},
{
"id": "flow-test4-ai-agent-tools",
"prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nCreate a customer support flow with an AI agent:\n\nSTEP 1: Receive customer query from input (customer_id string, query_text string)\nSTEP 2: Fetch customer profile and order history (mock data based on customer_id)\nSTEP 3: Use an AI agent to handle the customer query. The agent should have access to these tools:\n - lookup_order: Takes order_id, returns order details (mock data)\n - check_refund_eligibility: Takes order_id, returns eligibility status and reason\n - create_support_ticket: Takes description and priority (low/medium/high), returns ticket_id\n - search_faq: Takes search_query, returns relevant FAQ answers\n The agent should use the customer profile context and respond helpfully.\nSTEP 4: Log the interaction to audit trail (customer_id, query, response summary)\nSTEP 5: Return the agent's response and any actions taken",
"expected": "ai_evals/fixtures/frontend/flow/expected/test4.json"
},
{
"id": "flow-test5-simple-modification",
"prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nModify this existing flow to add error handling:\n- Add a new step after process_data called \"validate_data\" to validate the processed data\n- The validation step should check if the data array is not empty\n- If validation fails (empty array), it should return an error object with message \"No data to save\"\n- If validation passes, return the data for the next step\n- Update save_results to handle the validation result appropriately",
"initial": "ai_evals/fixtures/frontend/flow/initial/test5_initial.json",
"expected": "ai_evals/fixtures/frontend/flow/expected/test5_modify_simple.json"
},
{
"id": "flow-test6-branching-in-loop",
"prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nModify the order processing loop to handle different order types:\n- Inside the loop_orders, replace the simple process_order step with branching based on order.type\n- For type \"express\": add a step called handle_express that marks as priority and calculates express shipping cost ($15.99)\n- For type \"standard\": add a step called handle_standard that calculates standard shipping cost ($5.99)\n- For type \"pickup\": add a step called handle_pickup that marks as no shipping required (cost $0)\n- Move the original process_order step to the default branch for unknown order types\n- Each branch step should return the orderId, shipping cost, and shipping type",
"initial": "ai_evals/fixtures/frontend/flow/initial/test6_initial.json",
"expected": "ai_evals/fixtures/frontend/flow/expected/test6_modify_medium.json"
},
{
"id": "flow-test7-parallel-refactor",
"prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nRefactor this flow for better performance by parallelizing the enrichment steps:\n- The three enrichment steps (enrich_price, enrich_inventory, enrich_reviews) currently run sequentially\n- Wrap them in a parallel branch (branchall) called \"parallel_enrichment\" so they run concurrently\n- Each enrichment step should include basic error handling with try/catch that returns a fallback value if it fails\n- Update the combine_data step to receive results from the parallel branch (results.parallel_enrichment returns an array of branch results)\n- The combine_data step should check if any enrichment used a fallback value and set a hasFallbacks flag\n- Keep get_item as the first step and return_result as the last step unchanged",
"initial": "ai_evals/fixtures/frontend/flow/initial/test7_initial.json",
"expected": "ai_evals/fixtures/frontend/flow/expected/test7_modify_complex.json"
}
]

167
ai_evals/cases/frontend/app.json
View File

@@ -1,167 +0,0 @@
[
{
"id": "app-test1-counter-create",
"surface": "frontend-app",
"title": "test1: creates a simple counter app",
"user_prompt": "Create a counter app with increment/decrement buttons",
"initial_state": {},
"workspace_context": {},
"artifact_checks": {},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"app",
"create"
]
},
{
"id": "app-test2-counter-reset",
"surface": "frontend-app",
"title": "test2: modifies existing counter app to add reset button",
"user_prompt": "Add a reset button that sets the counter back to 0",
"initial_state": {
"app_fixture_path": "ai_evals/fixtures/frontend/app/initial/test1_counter_app"
},
"workspace_context": {},
"artifact_checks": {},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"app",
"modify"
]
},
{
"id": "app-test3-shopping-cart-quantity",
"surface": "frontend-app",
"title": "test3: shopping cart - add quantity selector",
"user_prompt": "Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items",
"initial_state": {
"app_fixture_path": "ai_evals/fixtures/frontend/app/initial/shopping_cart"
},
"workspace_context": {},
"artifact_checks": {},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"app",
"modify"
]
},
{
"id": "app-test4-shopping-cart-discount",
"surface": "frontend-app",
"title": "test4: shopping cart - add discount code",
"user_prompt": "Add a discount code input field in the cart. When the code \"SAVE10\" is entered, apply a 10% discount to the total",
"initial_state": {
"app_fixture_path": "ai_evals/fixtures/frontend/app/initial/shopping_cart"
},
"workspace_context": {},
"artifact_checks": {},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"app",
"modify"
]
},
{
"id": "app-test5-file-manager-search",
"surface": "frontend-app",
"title": "test5: file manager - add search bar",
"user_prompt": "Add a search bar in the toolbar that filters files and folders by name as the user types",
"initial_state": {
"app_fixture_path": "ai_evals/fixtures/frontend/app/initial/file_manager"
},
"workspace_context": {},
"artifact_checks": {},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"app",
"modify"
]
},
{
"id": "app-test6-file-manager-details",
"surface": "frontend-app",
"title": "test6: file manager - show file details",
"user_prompt": "Show file size (formatted as KB/MB) and modified date in the file list for each item",
"initial_state": {
"app_fixture_path": "ai_evals/fixtures/frontend/app/initial/file_manager"
},
"workspace_context": {},
"artifact_checks": {},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"app",
"modify"
]
},
{
"id": "app-test7-file-manager-select-all",
"surface": "frontend-app",
"title": "test7: file manager - add select all checkbox",
"user_prompt": "Add a \"Select All\" checkbox in the file list header and individual checkboxes for each file. Add a \"Delete Selected\" button that appears when items are selected",
"initial_state": {
"app_fixture_path": "ai_evals/fixtures/frontend/app/initial/file_manager"
},
"workspace_context": {},
"artifact_checks": {},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"app",
"modify"
]
},
{
"id": "app-test8-quiz-create",
"surface": "frontend-app",
"title": "test8: create quiz app from scratch",
"user_prompt": "Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct.",
"initial_state": {},
"workspace_context": {},
"artifact_checks": {},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"app",
"create"
]
},
{
"id": "app-test9-recipe-book-create",
"surface": "frontend-app",
"title": "test9: create recipe book from scratch",
"user_prompt": "Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes.",
"initial_state": {},
"workspace_context": {},
"artifact_checks": {},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"app",
"create"
]
}
]

141
ai_evals/cases/frontend/flow.json
View File

@@ -1,141 +0,0 @@
[
{
"id": "flow-test1-user-role-actions",
"surface": "frontend-flow",
"title": "test1: user role-based actions with loop and branches",
"user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Fetch mock users from api\nSTEP 2: Filter only active users:\nSTEP 3: Loop on all users\nSTEP 4: Do branches based on user's role, do different action based on that. Roles are admin, user, moderator\nSTEP 5: Return action taken for each user",
"initial_state": {},
"workspace_context": {},
"artifact_checks": {
"expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test1.json"
},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"flow",
"create"
]
},
{
"id": "flow-test2-order-processing",
"surface": "frontend-flow",
"title": "test2: e-commerce order processing with inventory check and branching",
"user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Receive order data from input (order has items array with name/price/quantity, customer_email, shipping_address)\nSTEP 2: Validate order - check all items have valid price > 0 and quantity > 0, return validation result\nSTEP 3: Calculate order total with 8% tax rate\nSTEP 4: Check inventory for each item (loop through items, return mock availability)\nSTEP 5: Branch based on inventory - if all items available, create shipment record; otherwise create backorder record\nSTEP 6: Send confirmation (mock email to customer_email)\nSTEP 7: Return final order summary with status",
"initial_state": {},
"workspace_context": {},
"artifact_checks": {
"expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test2.json"
},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"flow",
"create"
]
},
{
"id": "flow-test3-data-pipeline",
"surface": "frontend-flow",
"title": "test3: data pipeline with parallel processing and quality-based routing",
"user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nSTEP 1: Fetch list of data sources from configuration (return mock array of 3 source objects with id and url)\nSTEP 2: For each data source in parallel:\n - Fetch raw data from the source (mock fetch returning sample records)\n - Transform/clean the data (filter out invalid entries)\n - Validate the transformed data (return validation score 0-100)\nSTEP 3: Aggregate all validated data into single dataset with combined records\nSTEP 4: Calculate overall data quality score (average of all validation scores)\nSTEP 5: Branch based on quality score:\n - If score >= 90: Store in primary database and return success\n - If score >= 70 and < 90: Store in secondary database with warning flag\n - If score < 70: Store in quarantine and send alert\nSTEP 6: Return processing report with statistics (total records, quality score, destination)",
"initial_state": {},
"workspace_context": {},
"artifact_checks": {
"expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test3.json"
},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"flow",
"create"
]
},
{
"id": "flow-test4-ai-agent-tools",
"surface": "frontend-flow",
"title": "test4: AI agent with tools for customer support",
"user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nCreate a customer support flow with an AI agent:\n\nSTEP 1: Receive customer query from input (customer_id string, query_text string)\nSTEP 2: Fetch customer profile and order history (mock data based on customer_id)\nSTEP 3: Use an AI agent to handle the customer query. The agent should have access to these tools:\n - lookup_order: Takes order_id, returns order details (mock data)\n - check_refund_eligibility: Takes order_id, returns eligibility status and reason\n - create_support_ticket: Takes description and priority (low/medium/high), returns ticket_id\n - search_faq: Takes search_query, returns relevant FAQ answers\n The agent should use the customer profile context and respond helpfully.\nSTEP 4: Log the interaction to audit trail (customer_id, query, response summary)\nSTEP 5: Return the agent's response and any actions taken",
"initial_state": {},
"workspace_context": {},
"artifact_checks": {
"expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test4.json"
},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"flow",
"create"
]
},
{
"id": "flow-test5-simple-modification",
"surface": "frontend-flow",
"title": "test5: simple modification - add validation step to existing flow",
"user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nModify this existing flow to add error handling:\n- Add a new step after process_data called \"validate_data\" to validate the processed data\n- The validation step should check if the data array is not empty\n- If validation fails (empty array), it should return an error object with message \"No data to save\"\n- If validation passes, return the data for the next step\n- Update save_results to handle the validation result appropriately",
"initial_state": {
"flow_path": "ai_evals/fixtures/frontend/flow/initial/test5_initial.json"
},
"workspace_context": {},
"artifact_checks": {
"expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test5_modify_simple.json"
},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"flow",
"modify"
]
},
{
"id": "flow-test6-branching-in-loop",
"surface": "frontend-flow",
"title": "test6: medium modification - add branching inside existing loop",
"user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nModify the order processing loop to handle different order types:\n- Inside the loop_orders, replace the simple process_order step with branching based on order.type\n- For type \"express\": add a step called handle_express that marks as priority and calculates express shipping cost ($15.99)\n- For type \"standard\": add a step called handle_standard that calculates standard shipping cost ($5.99)\n- For type \"pickup\": add a step called handle_pickup that marks as no shipping required (cost $0)\n- Move the original process_order step to the default branch for unknown order types\n- Each branch step should return the orderId, shipping cost, and shipping type",
"initial_state": {
"flow_path": "ai_evals/fixtures/frontend/flow/initial/test6_initial.json"
},
"workspace_context": {},
"artifact_checks": {
"expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test6_modify_medium.json"
},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"flow",
"modify"
]
},
{
"id": "flow-test7-parallel-refactor",
"surface": "frontend-flow",
"title": "test7: complex modification - refactor sequential to parallel execution",
"user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE, IF WE NEED RETURN VALUES RETURN EXAMPLE VALUES\n\nRefactor this flow for better performance by parallelizing the enrichment steps:\n- The three enrichment steps (enrich_price, enrich_inventory, enrich_reviews) currently run sequentially\n- Wrap them in a parallel branch (branchall) called \"parallel_enrichment\" so they run concurrently\n- Each enrichment step should include basic error handling with try/catch that returns a fallback value if it fails\n- Update the combine_data step to receive results from the parallel branch (results.parallel_enrichment returns an array of branch results)\n- The combine_data step should check if any enrichment used a fallback value and set a hasFallbacks flag\n- Keep get_item as the first step and return_result as the last step unchanged",
"initial_state": {
"flow_path": "ai_evals/fixtures/frontend/flow/initial/test7_initial.json"
},
"workspace_context": {},
"artifact_checks": {
"expected_flow_path": "ai_evals/fixtures/frontend/flow/expected/test7_modify_complex.json"
},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"flow",
"modify"
]
}
]

23
ai_evals/cases/frontend/script.json
View File

@@ -1,23 +0,0 @@
[
{
"id": "script-test1-greet-user",
"surface": "frontend-script",
"title": "test1: create a greeting script",
"user_prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE.\n\nUpdate the current bun script so it exports `main(name: string)` and returns the plain string `Hello, ${name}!`.\nDo not return an object or array.\nDo not add external dependencies.",
"initial_state": {
"script_path": "ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json"
},
"workspace_context": {},
"artifact_checks": {
"expected_script_path": "ai_evals/fixtures/frontend/script/expected/test1_greet_user.json"
},
"judge_rubric": {
"min_score": 80
},
"tags": [
"frontend",
"script",
"modify"
]
}
]

8
ai_evals/cases/script.json Normal file
View File

@@ -0,0 +1,8 @@
[
{
"id": "script-test1-greet-user",
"prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE.\n\nUpdate the current bun script so it exports `main(name: string)` and returns the plain string `Hello, ${name}!`.\nDo not return an object or array.\nDo not add external dependencies.",
"initial": "ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json",
"expected": "ai_evals/fixtures/frontend/script/expected/test1_greet_user.json"
}
]

116
ai_evals/cli/README.md
View File

@@ -1,116 +0,0 @@
# Benchmark CLI
The benchmark CLI is built around saved local results.
The normal loop is:
1. run the suite on the current checkout
2. make your change
3. run again
4. diff the two saved results
## Commands
List cases:
```bash
cd ai_evals
bun run cli -- list-cases
bun run cli -- list-cases --surface flow
```
Run the current checkout and save a result under `ai_evals/results/`:
```bash
cd ai_evals
bun run cli -- run --surface flow --runs 3
bun run cli -- run --surface cli --runs 3
```
Diff two saved results:
```bash
cd ai_evals
bun run cli -- diff-results ai_evals/results/before.json ai_evals/results/after.json
```
Show recent official history:
```bash
cd ai_evals
bun run cli -- history --limit 10
```
Promote one local result into official history:
```bash
cd ai_evals
bun run cli -- promote-result ai_evals/results/latest.json --label main
```
## Frontend Workflow
If you are improving frontend flow/app/script chat, use the surface directly:
```bash
cd ai_evals
bun run cli -- run --surface flow --runs 3
```
Optional frontend overrides:
- `--provider anthropic|openai`
- `--model <model>`
- `--system-prompt-file <path>` to fully replace the system prompt
- `--append-system-prompt-file <path>` to append extra instructions to the
default system prompt
Example:
```bash
cd ai_evals
bun run cli -- run --surface flow --append-system-prompt-file ./prompt-experiment.md --runs 3
```
The benchmark user prompt still comes from the case manifest. These flags are
for current-checkout experiments, not for a checked-in variant system.
## CLI Guidance Workflow
If you are improving CLI skills or project guidance, run the `cli` surface:
```bash
cd ai_evals
bun run cli -- run --surface cli --runs 3
```
Optional CLI overrides:
- `--skills-source <path>`
- `--agents-source <path>`
- `--claude-source <path>`
Example:
```bash
cd ai_evals
bun run cli -- run --surface cli --skills-source ./system_prompts/auto-generated/skills --runs 3
```
For debugging one CLI case locally, keep the final temp workspace:
```bash
cd ai_evals
bun run cli -- run --surface cli --case bun-hello-script --runs 1 --keep-workspace
```
## Result Files
Each `run` command writes one JSON result file containing:
- run metadata
- aggregate metrics
- per-case summaries
- per-attempt details
Those files are meant for local comparison and are ignored by git.

942
ai_evals/cli/index.ts
View File

File diff suppressed because it is too large Load Diff

313
ai_evals/cli/results.ts
View File

@@ -1,313 +0,0 @@
import { mkdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
export type SurfaceName = "cli" | "flow" | "app" | "script";
export type CanonicalSurfaceName =
| "cli"
| "frontend-flow"
| "frontend-app"
| "frontend-script";
export interface AttemptSummary {
attempt: number;
passed: boolean;
durationMs: number;
assistantMessageCount: number;
toolCallCount: number;
skillInvocationCount: number;
skillsInvoked: string[];
toolsUsed: string[];
checks: Array<{ name: string; passed: boolean; required?: boolean }>;
requiredFailedChecks: string[];
expectedFiles: Array<{ path: string; exists: boolean }>;
pathSignature: string;
judgeScore: number | null;
error: string | null;
}
export interface AggregateMetrics {
totalRuns: number;
passedRuns: number;
passRate: number;
averageDurationMs: number;
medianDurationMs: number;
latencyPerSuccessMs: number;
averageAssistantMessages: number;
averageToolCalls: number;
averageSkillInvocations: number;
distinctSkillsInvoked: string[];
distinctToolsUsed: string[];
requiredFailureCounts: Array<{ name: string; count: number }>;
judgeScoreMean: number | null;
judgeScoreMedian: number | null;
judgeScoreP10: number | null;
}
export interface BenchmarkCaseResult extends AggregateMetrics {
caseId: string;
pathConsistency: number;
attempts: AttemptSummary[];
}
export interface BenchmarkRunResult extends AggregateMetrics {
version: "ai-evals-run-v1";
createdAt: string;
gitSha: string;
label: string;
surface: SurfaceName;
canonicalSurface: CanonicalSurfaceName;
provider: string;
model: string;
judgeModel: string | null;
runs: number;
totalCases: number;
fullyPassedCases: number;
caseIds: string[];
config: Record<string, unknown>;
caseResults: BenchmarkCaseResult[];
}
export interface FailureDelta {
name: string;
before: number;
after: number;
delta: number;
}
export interface CaseDelta {
caseId: string;
beforePassRate: number;
afterPassRate: number;
deltaPassRate: number;
beforeFailures: string[];
afterFailures: string[];
}
export interface BenchmarkRunDiff {
surface: SurfaceName;
beforeLabel: string;
afterLabel: string;
summary: {
fullyPassedCasesDelta: number;
passRateDelta: number;
judgeScoreMeanDelta: number | null;
averageDurationMsDelta: number;
};
improvedCases: CaseDelta[];
regressedCases: CaseDelta[];
failureDeltas: FailureDelta[];
}
const RESULTS_DIR = fileURLToPath(new URL("../results", import.meta.url));
export async function writeBenchmarkRunResult(
result: BenchmarkRunResult,
outputPath?: string
): Promise<string> {
const resolvedPath = outputPath
? path.resolve(outputPath)
: path.join(
RESULTS_DIR,
`${slugifyTimestamp(result.createdAt)}__${slugify(result.surface)}__${slugify(result.label)}__${result.gitSha.slice(0, 12)}.json`
);
await mkdir(path.dirname(resolvedPath), { recursive: true });
await writeFile(resolvedPath, JSON.stringify(result, null, 2) + "\n", "utf8");
return resolvedPath;
}
export async function readBenchmarkRunResult(
inputPath: string
): Promise<BenchmarkRunResult> {
const resolvedPath = path.resolve(inputPath);
const raw = await readFile(resolvedPath, "utf8");
const parsed = JSON.parse(raw) as BenchmarkRunResult;
if (parsed.version !== "ai-evals-run-v1") {
throw new Error(`Unsupported benchmark result version in ${resolvedPath}`);
}
if (!Array.isArray(parsed.caseResults) || parsed.caseResults.length === 0) {
throw new Error(`Benchmark result ${resolvedPath} has no case results`);
}
return parsed;
}
export function buildBenchmarkRunDiff(
before: BenchmarkRunResult,
after: BenchmarkRunResult
): BenchmarkRunDiff {
if (before.surface !== after.surface) {
throw new Error(
`Cannot diff ${before.surface} against ${after.surface}`
);
}
const beforeCases = new Map(before.caseResults.map((entry) => [entry.caseId, entry]));
const afterCases = new Map(after.caseResults.map((entry) => [entry.caseId, entry]));
const sharedCaseIds = [...beforeCases.keys()].filter((caseId) => afterCases.has(caseId));
const caseDeltas = sharedCaseIds.map((caseId) => {
const beforeCase = beforeCases.get(caseId)!;
const afterCase = afterCases.get(caseId)!;
return {
caseId,
beforePassRate: beforeCase.passRate,
afterPassRate: afterCase.passRate,
deltaPassRate: afterCase.passRate - beforeCase.passRate,
beforeFailures: beforeCase.requiredFailureCounts.map((entry) => entry.name),
afterFailures: afterCase.requiredFailureCounts.map((entry) => entry.name)
} satisfies CaseDelta;
});
const failureCounts = new Map<string, { before: number; after: number }>();
for (const entry of before.requiredFailureCounts) {
failureCounts.set(entry.name, { before: entry.count, after: 0 });
}
for (const entry of after.requiredFailureCounts) {
const previous = failureCounts.get(entry.name) ?? { before: 0, after: 0 };
failureCounts.set(entry.name, { ...previous, after: entry.count });
}
return {
surface: before.surface,
beforeLabel: before.label,
afterLabel: after.label,
summary: {
fullyPassedCasesDelta: after.fullyPassedCases - before.fullyPassedCases,
passRateDelta: after.passRate - before.passRate,
judgeScoreMeanDelta:
before.judgeScoreMean === null || after.judgeScoreMean === null
? null
: after.judgeScoreMean - before.judgeScoreMean,
averageDurationMsDelta: after.averageDurationMs - before.averageDurationMs
},
improvedCases: caseDeltas
.filter((entry) => entry.deltaPassRate > 0)
.sort((left, right) => right.deltaPassRate - left.deltaPassRate || left.caseId.localeCompare(right.caseId)),
regressedCases: caseDeltas
.filter((entry) => entry.deltaPassRate < 0)
.sort((left, right) => left.deltaPassRate - right.deltaPassRate || left.caseId.localeCompare(right.caseId)),
failureDeltas: [...failureCounts.entries()]
.map(([name, counts]) => ({
name,
before: counts.before,
after: counts.after,
delta: counts.after - counts.before
}))
.filter((entry) => entry.delta !== 0)
.sort((left, right) => Math.abs(right.delta) - Math.abs(left.delta) || left.name.localeCompare(right.name))
};
}
export function buildOfficialRunFromResult(
result: BenchmarkRunResult,
labelOverride?: string
) {
const flakeRate =
result.caseResults.length === 0
? 0
: result.caseResults.filter(
(entry) => entry.passRate > 0 && entry.passRate < 1
).length / result.caseResults.length;
const pathConsistency = average(
result.caseResults.map((entry) => entry.pathConsistency)
);
const qualityScore = result.passRate * 100;
const efficiencyScore = computeEfficiencyScore(result);
return {
timestamp: result.createdAt,
git_sha: result.gitSha,
suite_version: "cli-benchmark-v1",
scoring_version: "cli-deterministic-v1",
surface: result.canonicalSurface,
label: labelOverride ?? result.label,
provider: result.provider,
model: result.model,
judge_model: result.judgeModel,
runs_per_case: result.runs,
case_count: result.caseResults.length,
metrics: {
quality: {
pass_rate: result.passRate,
deterministic_pass_rate: result.passRate,
judge_score_mean: result.judgeScoreMean ?? 0,
judge_score_median: result.judgeScoreMedian ?? 0,
judge_score_p10: result.judgeScoreP10 ?? 0,
quality_score: qualityScore
},
reliability: {
runs_per_case: result.runs,
flake_rate: flakeRate,
path_consistency: pathConsistency
},
efficiency: {
latency_ms_mean: result.averageDurationMs,
latency_ms_median: result.medianDurationMs,
tokens_total_mean: 0,
tool_calls_mean: result.averageToolCalls,
iterations_mean: result.averageAssistantMessages,
estimated_cost_mean: 0,
cost_per_success: 0,
latency_per_success: result.latencyPerSuccessMs,
efficiency_score: efficiencyScore,
value_score: (qualityScore * efficiencyScore) / 100
}
},
cases: result.caseResults.map((entry) => ({
id: entry.caseId,
pass_rate: entry.passRate,
average_duration_ms: entry.averageDurationMs,
median_duration_ms: entry.medianDurationMs,
latency_per_success_ms: entry.latencyPerSuccessMs,
average_assistant_messages: entry.averageAssistantMessages,
average_tool_calls: entry.averageToolCalls,
average_skill_invocations: entry.averageSkillInvocations,
path_consistency: entry.pathConsistency,
distinct_skills_invoked: entry.distinctSkillsInvoked,
distinct_tools_used: entry.distinctToolsUsed,
required_failure_counts: entry.requiredFailureCounts
}))
};
}
export function formatSurfaceLabel(surface: SurfaceName | CanonicalSurfaceName): string {
if (surface === "frontend-flow") {
return "flow";
}
if (surface === "frontend-app") {
return "app";
}
if (surface === "frontend-script") {
return "script";
}
return surface;
}
function computeEfficiencyScore(result: BenchmarkRunResult): number {
const latencyFactor = 1 / (1 + result.averageDurationMs / 20000);
const toolFactor = 1 / (1 + result.averageToolCalls / 10);
const iterationFactor = 1 / (1 + result.averageAssistantMessages / 10);
return ((latencyFactor + toolFactor + iterationFactor) / 3) * 100;
}
function average(values: number[]): number {
if (values.length === 0) {
return 0;
}
return values.reduce((sum, value) => sum + value, 0) / values.length;
}
function slugifyTimestamp(value: string): string {
return value.replaceAll(":", "-").replaceAll(".", "-");
}
function slugify(value: string): string {
return value
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "") || "default";
}

62
ai_evals/core/cases.ts Normal file
View File

@@ -0,0 +1,62 @@
import { readFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import type { EvalCase, EvalMode } from "./types";
const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url));
const CASES_DIR = path.join(REPO_ROOT, "ai_evals", "cases");
interface RawEvalCase {
id: string;
prompt: string;
initial?: string;
expected?: string;
}
export function getRepoRoot(): string {
return REPO_ROOT;
}
export function getAiEvalsRoot(): string {
return path.join(REPO_ROOT, "ai_evals");
}
export async function loadCases(mode: EvalMode): Promise<EvalCase[]> {
const filePath = path.join(CASES_DIR, `${mode}.json`);
const raw = await readFile(filePath, "utf8");
const parsed = JSON.parse(raw) as RawEvalCase[];
return parsed.map((entry) => ({
id: entry.id,
prompt: entry.prompt,
initialPath: resolveFixturePath(entry.initial),
expectedPath: resolveFixturePath(entry.expected),
}));
}
export async function loadSelectedCases(
mode: EvalMode,
selectedIds: string[]
): Promise<EvalCase[]> {
const allCases = await loadCases(mode);
if (selectedIds.length === 0) {
return allCases;
}
const caseMap = new Map(allCases.map((entry) => [entry.id, entry]));
const missing = selectedIds.filter((id) => !caseMap.has(id));
if (missing.length > 0) {
throw new Error(
`Unknown ${mode} case${missing.length === 1 ? "" : "s"}: ${missing.join(", ")}`
);
}
return selectedIds.map((id) => caseMap.get(id)!);
}
function resolveFixturePath(value: string | undefined): string | undefined {
if (!value) {
return undefined;
}
return path.isAbsolute(value) ? value : path.join(REPO_ROOT, value);
}

67
ai_evals/core/files.ts Normal file
View File

@@ -0,0 +1,67 @@
import { access, copyFile, mkdir, readdir, readFile } from "node:fs/promises";
import path from "node:path";
export async function exists(filePath: string): Promise<boolean> {
try {
await access(filePath);
return true;
} catch {
return false;
}
}
export async function readJsonFile<T>(filePath: string): Promise<T> {
const raw = await readFile(filePath, "utf8");
return JSON.parse(raw) as T;
}
export async function readDirectoryFiles(
rootDir: string,
options: {
ignore?: Set<string>;
} = {}
): Promise<Record<string, string>> {
const files: Record<string, string> = {};
await walkDirectory(rootDir, "", files, options.ignore ?? new Set());
return files;
}
export async function copyDirectory(sourceDir: string, targetDir: string): Promise<void> {
const entries = await readdir(sourceDir, { withFileTypes: true });
await mkdir(targetDir, { recursive: true });
for (const entry of entries) {
const sourcePath = path.join(sourceDir, entry.name);
const targetPath = path.join(targetDir, entry.name);
if (entry.isDirectory()) {
await copyDirectory(sourcePath, targetPath);
continue;
}
await mkdir(path.dirname(targetPath), { recursive: true });
await copyFile(sourcePath, targetPath);
}
}
async function walkDirectory(
absoluteDir: string,
relativeDir: string,
output: Record<string, string>,
ignore: Set<string>
): Promise<void> {
const entries = await readdir(absoluteDir, { withFileTypes: true });
for (const entry of entries) {
const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name;
if (ignore.has(relativePath) || ignore.has(entry.name)) {
continue;
}
const absolutePath = path.join(absoluteDir, entry.name);
if (entry.isDirectory()) {
await walkDirectory(absolutePath, relativePath, output, ignore);
continue;
}
output[relativePath] = await readFile(absolutePath, "utf8");
}
}

134
ai_evals/core/judge.ts Normal file
View File

@@ -0,0 +1,134 @@
import Anthropic from "@anthropic-ai/sdk";
import type { EvalMode, JudgeResult } from "./types";
export const DEFAULT_JUDGE_MODEL = "claude-sonnet-4-6";
const JUDGE_TOOL_NAME = "submit_judgement";
export async function judgeOutput(input: {
mode: EvalMode;
prompt: string;
initial?: unknown;
expected?: unknown;
actual: unknown;
model?: string;
}): Promise<JudgeResult> {
const apiKey = process.env.ANTHROPIC_API_KEY;
if (!apiKey) {
return {
success: false,
score: 0,
summary: "Judge unavailable",
error: "ANTHROPIC_API_KEY is not set",
};
}
const client = new Anthropic({ apiKey });
const model = input.model ?? DEFAULT_JUDGE_MODEL;
const system = [
"You evaluate benchmark outputs for Windmill AI generation.",
"Deterministic checks already run separately. Focus on whether the final output satisfies the user request.",
"If expected state is provided, treat it as a strong reference and reward semantically equivalent outputs.",
"Be strict about missing requested functionality.",
`Always respond by calling the ${JUDGE_TOOL_NAME} tool exactly once.`,
].join("\n\n");
const user = [
`Mode: ${input.mode}`,
"",
"User prompt:",
input.prompt,
"",
"Initial state:",
formatJsonBlock(input.initial),
"",
"Expected state:",
formatJsonBlock(input.expected),
"",
"Actual result:",
formatJsonBlock(input.actual),
].join("\n");
try {
const response = await client.messages.create({
model,
max_tokens: 1024,
temperature: 0,
system,
messages: [{ role: "user", content: user }],
tools: [
{
name: JUDGE_TOOL_NAME,
description: "Submit the benchmark judgement as structured data.",
input_schema: {
type: "object",
properties: {
score: {
type: "integer",
minimum: 0,
maximum: 100,
},
summary: {
type: "string",
},
},
required: ["score", "summary"],
},
},
],
tool_choice: {
type: "tool",
name: JUDGE_TOOL_NAME,
disable_parallel_tool_use: true,
},
});
const toolUseBlock = response.content.find(
(block): block is Anthropic.ToolUseBlock =>
block.type === "tool_use" && block.name === JUDGE_TOOL_NAME
);
if (!toolUseBlock) {
return {
success: false,
score: 0,
summary: "Judge returned no tool output",
error: "Expected structured tool output from judge",
};
}
const parsed = toolUseBlock.input as {
score: number;
summary: string;
};
return {
success: true,
score: normalizeScore(parsed.score),
summary: parsed.summary,
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
success: false,
score: 0,
summary: "Judge failed",
error: message,
};
}
}
function formatJsonBlock(value: unknown): string {
if (value === undefined) {
return "(none)";
}
return JSON.stringify(value, null, 2);
}
function normalizeScore(value: number): number {
if (!Number.isFinite(value)) {
return 0;
}
return Math.max(0, Math.min(100, Math.round(value)));
}

110
ai_evals/core/results.ts Normal file
View File

@@ -0,0 +1,110 @@
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { getAiEvalsRoot, getRepoRoot } from "./cases";
import type {
BenchmarkCaseResult,
BenchmarkRunResult,
EvalMode,
} from "./types";
export async function writeRunResult(
result: BenchmarkRunResult,
outputPath?: string
): Promise<string> {
const targetPath =
outputPath ?? path.join(getAiEvalsRoot(), "results", defaultFileName(result.mode));
await mkdir(path.dirname(targetPath), { recursive: true });
await writeFile(targetPath, JSON.stringify(result, null, 2) + "\n", "utf8");
return targetPath;
}
export function buildRunResult(input: {
mode: EvalMode;
runs: number;
runModel: string | null;
judgeModel: string | null;
caseResults: BenchmarkCaseResult[];
}): BenchmarkRunResult {
const attemptCount = input.caseResults.reduce((sum, entry) => sum + entry.attempts.length, 0);
const passedAttempts = input.caseResults.reduce(
(sum, entry) => sum + entry.attempts.filter((attempt) => attempt.passed).length,
0
);
const durationTotal = input.caseResults.reduce(
(sum, entry) => sum + entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0),
0
);
return {
version: 1,
mode: input.mode,
createdAt: new Date().toISOString(),
gitSha: getGitSha(),
runs: input.runs,
runModel: input.runModel,
judgeModel: input.judgeModel,
caseCount: input.caseResults.length,
attemptCount,
passedAttempts,
passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount,
averageDurationMs: attemptCount === 0 ? 0 : durationTotal / attemptCount,
cases: input.caseResults,
};
}
export function formatRunSummary(result: BenchmarkRunResult): string {
const lines = [
`${result.mode} benchmark complete`,
`Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`,
`Average duration: ${Math.round(result.averageDurationMs)}ms`,
];
const failures = collectFailures(result);
if (failures.length > 0) {
lines.push("Failures:");
for (const entry of failures.slice(0, 10)) {
lines.push(`- ${entry}`);
}
}
return lines.join("\n");
}
function collectFailures(result: BenchmarkRunResult): string[] {
const failures: string[] = [];
for (const caseResult of result.cases) {
for (const attempt of caseResult.attempts) {
if (attempt.passed) {
continue;
}
const failedChecks = attempt.checks.filter((check) => !check.passed).map((check) => check.name);
failures.push(
`${caseResult.id} attempt ${attempt.attempt}: ${failedChecks.join(", ") || attempt.error || "failed"}`
);
}
}
return failures;
}
function defaultFileName(mode: EvalMode): string {
return `${new Date().toISOString().replaceAll(":", "-")}__${mode}.json`;
}
function getGitSha(): string | null {
try {
return execFileSync("git", ["rev-parse", "HEAD"], {
cwd: getRepoRoot(),
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"],
}).trim();
} catch {
return null;
}
}
function formatPercent(value: number): string {
return `${(value * 100).toFixed(1)}%`;
}

202
ai_evals/core/runSuite.ts Normal file
View File

@@ -0,0 +1,202 @@
import { judgeOutput, DEFAULT_JUDGE_MODEL } from "./judge";
import type {
BenchmarkAttemptResult,
BenchmarkCaseResult,
BenchmarkCheck,
EvalCase,
FrontendBenchmarkProgressEvent,
ModeRunner,
} from "./types";
export async function runSuite<TInitial, TExpected, TActual>(input: {
modeRunner: ModeRunner<TInitial, TExpected, TActual>;
cases: EvalCase[];
runs: number;
runModel: string | null;
judgeModel?: string | null;
onProgress?: (event: FrontendBenchmarkProgressEvent) => void;
}): Promise<BenchmarkCaseResult[]> {
const judgeModel = input.judgeModel ?? DEFAULT_JUDGE_MODEL;
const concurrency = Math.max(1, input.modeRunner.concurrency);
const results = new Array<BenchmarkCaseResult>(input.cases.length);
let cursor = 0;
if (input.modeRunner.mode !== "cli") {
input.onProgress?.({
type: "run-start",
surface: input.modeRunner.mode,
totalCases: input.cases.length,
runs: input.runs,
concurrency,
});
}
async function worker(): Promise<void> {
while (true) {
const caseIndex = cursor++;
if (caseIndex >= input.cases.length) {
return;
}
const evalCase = input.cases[caseIndex];
results[caseIndex] = {
id: evalCase.id,
prompt: evalCase.prompt,
initialPath: evalCase.initialPath,
expectedPath: evalCase.expectedPath,
attempts: await runCaseAttempts({
caseIndex,
evalCase,
runs: input.runs,
judgeModel,
judgeThreshold: input.modeRunner.judgeThreshold ?? 80,
modeRunner: input.modeRunner,
totalCases: input.cases.length,
onProgress: input.onProgress,
}),
};
}
}
await Promise.all(
Array.from({ length: Math.min(concurrency, input.cases.length) }, () => worker())
);
return results;
}
async function runCaseAttempts<TInitial, TExpected, TActual>(input: {
caseIndex: number;
evalCase: EvalCase;
runs: number;
judgeModel: string;
judgeThreshold: number;
modeRunner: ModeRunner<TInitial, TExpected, TActual>;
totalCases: number;
onProgress?: (event: FrontendBenchmarkProgressEvent) => void;
}): Promise<BenchmarkAttemptResult[]> {
const attempts: BenchmarkAttemptResult[] = [];
for (let attempt = 1; attempt <= input.runs; attempt += 1) {
input.onProgress?.({
type: "attempt-start",
surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
caseId: input.evalCase.id,
caseNumber: input.caseIndex + 1,
totalCases: input.totalCases,
attempt,
runs: input.runs,
});
const startedAt = Date.now();
const initial = await input.modeRunner.loadInitial(input.evalCase.initialPath);
const expected = await input.modeRunner.loadExpected(input.evalCase.expectedPath);
try {
const run = await input.modeRunner.run(input.evalCase.prompt, initial);
const checks: BenchmarkCheck[] = [
buildCheck("run succeeded", run.success, run.error),
...input.modeRunner.validate({
prompt: input.evalCase.prompt,
initial,
expected,
actual: run.actual,
run,
}),
];
let judgeScore: number | null = null;
let judgeSummary: string | null = null;
if (run.success) {
const judge = await judgeOutput({
mode: input.modeRunner.mode,
prompt: input.evalCase.prompt,
initial,
expected,
actual: run.actual,
model: input.judgeModel,
});
judgeScore = judge.success ? judge.score : null;
judgeSummary = judge.summary;
checks.push(buildCheck("judge succeeded", judge.success, judge.error));
checks.push(
buildCheck(
`judge score >= ${input.judgeThreshold}`,
(judgeScore ?? 0) >= input.judgeThreshold,
judge.success ? `score=${judgeScore}` : judge.error
)
);
}
const attemptResult: BenchmarkAttemptResult = {
attempt,
passed: checks.every((check) => check.passed),
durationMs: Date.now() - startedAt,
assistantMessageCount: run.assistantMessageCount,
toolCallCount: run.toolCallCount,
toolsUsed: uniqueStrings(run.toolsUsed),
skillsInvoked: uniqueStrings(run.skillsInvoked),
checks,
judgeScore,
judgeSummary,
error: run.error ?? null,
};
input.onProgress?.({
type: "attempt-finish",
surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
caseId: input.evalCase.id,
caseNumber: input.caseIndex + 1,
totalCases: input.totalCases,
attempt,
runs: input.runs,
passed: attemptResult.passed,
durationMs: attemptResult.durationMs,
judgeScore: attemptResult.judgeScore,
error: attemptResult.error,
});
attempts.push(attemptResult);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const failedAttempt: BenchmarkAttemptResult = {
attempt,
passed: false,
durationMs: Date.now() - startedAt,
assistantMessageCount: 0,
toolCallCount: 0,
toolsUsed: [],
skillsInvoked: [],
checks: [buildCheck("run crashed", false, message)],
judgeScore: null,
judgeSummary: null,
error: message,
};
input.onProgress?.({
type: "attempt-finish",
surface: input.modeRunner.mode as Exclude<typeof input.modeRunner.mode, "cli">,
caseId: input.evalCase.id,
caseNumber: input.caseIndex + 1,
totalCases: input.totalCases,
attempt,
runs: input.runs,
passed: false,
durationMs: failedAttempt.durationMs,
judgeScore: null,
error: message,
});
attempts.push(failedAttempt);
}
}
return attempts;
}
function buildCheck(name: string, passed: boolean, details?: string): BenchmarkCheck {
return details ? { name, passed, details } : { name, passed };
}
function uniqueStrings(values: string[]): string[] {
return [...new Set(values)];
}

118
ai_evals/core/types.ts Normal file
View File

@@ -0,0 +1,118 @@
export const EVAL_MODES = ["cli", "flow", "script", "app"] as const;
export type EvalMode = (typeof EVAL_MODES)[number];
export interface EvalCase {
id: string;
prompt: string;
initialPath?: string;
expectedPath?: string;
}
export interface BenchmarkCheck {
name: string;
passed: boolean;
details?: string;
}
export interface JudgeResult {
success: boolean;
score: number;
summary: string;
error?: string;
}
export interface ModeRunOutput<TActual> {
success: boolean;
actual: TActual;
error?: string;
assistantMessageCount: number;
toolCallCount: number;
toolsUsed: string[];
skillsInvoked: string[];
}
export interface ModeRunner<TInitial, TExpected, TActual> {
mode: EvalMode;
concurrency: number;
judgeThreshold?: number;
loadInitial(path?: string): Promise<TInitial | undefined>;
loadExpected(path?: string): Promise<TExpected | undefined>;
run(prompt: string, initial: TInitial | undefined): Promise<ModeRunOutput<TActual>>;
validate(input: {
prompt: string;
initial: TInitial | undefined;
expected: TExpected | undefined;
actual: TActual;
run: ModeRunOutput<TActual>;
}): BenchmarkCheck[];
}
export interface BenchmarkAttemptResult {
attempt: number;
passed: boolean;
durationMs: number;
assistantMessageCount: number;
toolCallCount: number;
toolsUsed: string[];
skillsInvoked: string[];
checks: BenchmarkCheck[];
judgeScore: number | null;
judgeSummary: string | null;
error: string | null;
}
export interface BenchmarkCaseResult {
id: string;
prompt: string;
initialPath?: string;
expectedPath?: string;
attempts: BenchmarkAttemptResult[];
}
export interface BenchmarkRunResult {
version: 1;
mode: EvalMode;
createdAt: string;
gitSha: string | null;
runs: number;
runModel: string | null;
judgeModel: string | null;
caseCount: number;
attemptCount: number;
passedAttempts: number;
passRate: number;
averageDurationMs: number;
cases: BenchmarkCaseResult[];
}
export type FrontendBenchmarkProgressEvent =
| {
type: "run-start";
surface: Exclude<EvalMode, "cli">;
totalCases: number;
runs: number;
concurrency: number;
}
| {
type: "attempt-start";
surface: Exclude<EvalMode, "cli">;
caseId: string;
caseNumber: number;
totalCases: number;
attempt: number;
runs: number;
}
| {
type: "attempt-finish";
surface: Exclude<EvalMode, "cli">;
caseId: string;
caseNumber: number;
totalCases: number;
attempt: number;
runs: number;
passed: boolean;
durationMs: number;
judgeScore: number | null;
error: string | null;
};

287
ai_evals/core/validators.ts Normal file
View File

@@ -0,0 +1,287 @@
import ts from "typescript";
import type { BenchmarkCheck } from "./types";
export interface ScriptState {
path: string;
lang: string;
args?: Record<string, unknown>;
code: string;
}
export interface FlowState {
value?: {
modules?: Array<Record<string, unknown>>;
};
schema?: Record<string, unknown>;
}
export interface AppFilesState {
frontend: Record<string, string>;
backend: Record<string, AppRunnableState>;
}
export interface AppRunnableState {
type?: string;
name?: string;
path?: string;
inlineScript?: {
language?: string;
content?: string;
};
}
const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]);
export function validateScriptState(input: {
actual: ScriptState;
initial?: ScriptState;
expected?: ScriptState;
}): BenchmarkCheck[] {
const checks: BenchmarkCheck[] = [
check("script exports entrypoint", hasSupportedEntrypoint(input.actual.code)),
check("script has no syntax errors", getScriptSyntaxErrors(input.actual.code, input.actual.lang).length === 0),
];
if (input.expected) {
checks.push(
check(
"script path matches expected",
input.actual.path === input.expected.path,
`expected ${input.expected.path}, got ${input.actual.path}`
)
);
checks.push(
check(
"script language matches expected",
input.actual.lang === input.expected.lang,
`expected ${input.expected.lang}, got ${input.actual.lang}`
)
);
checks.push(
check(
"script code matches expected",
normalizeText(input.actual.code) === normalizeText(input.expected.code)
)
);
}
if (input.initial) {
checks.push(
check(
"script differs from initial",
normalizeText(input.actual.code) !== normalizeText(input.initial.code)
)
);
}
return checks;
}
export function validateFlowState(input: {
actual: FlowState;
expected?: FlowState;
}): BenchmarkCheck[] {
const actualModules = getFlowModules(input.actual);
const checks: BenchmarkCheck[] = [check("flow has modules", actualModules.length > 0)];
if (!input.expected) {
return checks;
}
const expectedModules = getFlowModules(input.expected);
const actualTypes = new Set(actualModules.map((module) => String(module.type ?? "")));
const expectedTypes = new Set(expectedModules.map((module) => String(module.type ?? "")));
const missingTypes = [...expectedTypes].filter((type) => type && !actualTypes.has(type));
const actualTopLevelIds = getTopLevelFlowModuleIds(input.actual);
const expectedTopLevelIds = getTopLevelFlowModuleIds(input.expected);
const missingIds = expectedTopLevelIds.filter((id) => !actualTopLevelIds.includes(id));
const expectedSchemaRootType = getSchemaRootType(input.expected.schema);
const actualSchemaRootType = getSchemaRootType(input.actual.schema);
checks.push(
check(
"flow includes expected module types",
missingTypes.length === 0,
missingTypes.length > 0 ? `missing: ${missingTypes.join(", ")}` : undefined
)
);
if (expectedSchemaRootType) {
checks.push(
check(
"flow schema root type matches expected",
actualSchemaRootType === expectedSchemaRootType,
`expected ${expectedSchemaRootType}, got ${actualSchemaRootType ?? "(missing)"}`
)
);
}
if (expectedTopLevelIds.length > 0) {
checks.push(
check(
"flow includes expected top-level step ids",
missingIds.length === 0,
missingIds.length > 0 ? `missing: ${missingIds.join(", ")}` : undefined
)
);
}
return checks;
}
export function validateAppState(input: {
actual: AppFilesState;
initial?: AppFilesState;
expected?: AppFilesState;
}): BenchmarkCheck[] {
const checks: BenchmarkCheck[] = [];
const frontendEntries = Object.entries(input.actual.frontend ?? {});
const backendEntries = Object.entries(input.actual.backend ?? {});
checks.push(check("app has frontend entrypoint", Boolean(input.actual.frontend["/index.tsx"])));
checks.push(check("app has non-empty frontend files", frontendEntries.some(([, content]) => content.trim().length > 0)));
checks.push(
check(
"backend inline scripts have entrypoints",
backendEntries.every(([, runnable]) => {
if (runnable.type !== "inline") {
return true;
}
return hasSupportedEntrypoint(runnable.inlineScript?.content ?? "");
})
)
);
if (input.initial) {
checks.push(check("app differs from initial", !appStatesEqual(input.actual, input.initial)));
}
if (input.expected) {
for (const [filePath, content] of Object.entries(input.expected.frontend)) {
checks.push(
check(
`frontend includes ${filePath}`,
normalizeText(input.actual.frontend[filePath] ?? "") === normalizeText(content)
)
);
}
for (const [runnableName, runnable] of Object.entries(input.expected.backend)) {
const actualRunnable = input.actual.backend[runnableName];
checks.push(check(`backend includes ${runnableName}`, Boolean(actualRunnable)));
if (actualRunnable && runnable.inlineScript?.content) {
checks.push(
check(
`${runnableName} code matches expected`,
normalizeText(actualRunnable.inlineScript?.content ?? "") ===
normalizeText(runnable.inlineScript.content)
)
);
}
}
}
return checks;
}
export function validateCliWorkspace(input: {
actualFiles: Record<string, string>;
expectedFiles?: Record<string, string>;
initialFiles?: Record<string, string>;
}): BenchmarkCheck[] {
const checks: BenchmarkCheck[] = [];
if (input.expectedFiles) {
for (const [filePath, expectedContent] of Object.entries(input.expectedFiles)) {
const actualContent = input.actualFiles[filePath];
checks.push(check(`creates ${filePath}`, actualContent !== undefined));
if (actualContent !== undefined) {
checks.push(
check(
`${filePath} contains expected content`,
actualContent.includes(expectedContent.trim())
)
);
}
}
}
if (input.initialFiles) {
checks.push(check("workspace differs from initial", !fileMapsEqual(input.actualFiles, input.initialFiles)));
}
return checks;
}
function check(name: string, passed: boolean, details?: string): BenchmarkCheck {
return details ? { name, passed, details } : { name, passed };
}
function normalizeText(value: string): string {
return value.replace(/\r\n/g, "\n").trim();
}
function hasSupportedEntrypoint(code: string): boolean {
return (
/export\s+(async\s+)?function\s+main\s*\(/.test(code) ||
/export\s+default\s+(async\s+)?function\s*\(/.test(code)
);
}
function getScriptSyntaxErrors(code: string, lang: string): string[] {
if (!TS_LIKE_LANGUAGES.has(lang)) {
return [];
}
const sourceFile = ts.createSourceFile("eval.ts", code, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
return sourceFile.parseDiagnostics.map((diagnostic) =>
ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")
);
}
function getFlowModules(flow: FlowState): Array<Record<string, unknown>> {
return Array.isArray(flow.value?.modules) ? flow.value.modules : [];
}
function getTopLevelFlowModuleIds(flow: FlowState): string[] {
return getFlowModules(flow)
.map((module) => module.id)
.filter((value): value is string => typeof value === "string");
}
function getSchemaRootType(schema: Record<string, unknown> | undefined): string | null {
if (!schema || typeof schema !== "object") {
return null;
}
const properties = (schema.properties ?? {}) as Record<string, unknown>;
const root = properties["root"];
if (!root || typeof root !== "object") {
return null;
}
return typeof (root as { type?: unknown }).type === "string"
? ((root as { type: string }).type)
: null;
}
function appStatesEqual(left: AppFilesState, right: AppFilesState): boolean {
return fileMapsEqual(left.frontend, right.frontend) && fileMapsEqual(stringifyBackend(left.backend), stringifyBackend(right.backend));
}
function stringifyBackend(backend: Record<string, AppRunnableState>): Record<string, string> {
const result: Record<string, string> = {};
for (const [key, value] of Object.entries(backend)) {
result[key] = JSON.stringify(value);
}
return result;
}
function fileMapsEqual(left: Record<string, string>, right: Record<string, string>): boolean {
const leftEntries = Object.entries(left).sort(([a], [b]) => a.localeCompare(b));
const rightEntries = Object.entries(right).sort(([a], [b]) => a.localeCompare(b));
if (leftEntries.length !== rightEntries.length) {
return false;
}
return leftEntries.every(([key, value], index) => {
const [otherKey, otherValue] = rightEntries[index];
return key === otherKey && normalizeText(value) === normalizeText(otherValue);
});
}

6
ai_evals/fixtures/cli/expected/bun-hello-flow/f/evals/hello__flow/flow.yaml Normal file
View File

@@ -0,0 +1,6 @@
value:
modules:
Add File: /home/farhad/windmill__worktrees/prompt-testing-plan/ai_evals/fixtures/cli/expected/bun-hello-flow/f/evals/hello__flow/hello.ts
export async function main(name: string) {
return { greeting: `Hello, ${name}!` };
}

3
ai_evals/fixtures/cli/expected/bun-hello-flow/f/evals/hello__flow/hello.ts Normal file
View File

@@ -0,0 +1,3 @@
export async function main(name: string) {
return { greeting: `Hello, ${name}!` };
}

3
ai_evals/fixtures/cli/expected/bun-hello-script/f/evals/hello.ts Normal file
View File

@@ -0,0 +1,3 @@
export async function main(name: string) {
return { greeting: `Hello, ${name}!` };
}

31
ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json Normal file
View File

@@ -0,0 +1,31 @@
{
"summary": "",
"value": {
"modules": [
{
"id": "sum_numbers",
"value": {
"type": "rawscript",
"language": "bun",
"content": "export async function main(a: number, b: number) {\n return a + b;\n}",
"input_transforms": {
"a": {
"type": "javascript",
"expr": "flow_input.a"
},
"b": {
"type": "javascript",
"expr": "flow_input.b"
}
}
}
}
]
},
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {},
"required": [],
"type": "object"
}
}

65
ai_evals/history/README.md
View File

@@ -1,65 +0,0 @@
# Benchmark History
This directory stores the git-tracked benchmark history for official AI eval
runs.
## Purpose
The history layer answers a different question than a one-off benchmark run:
> Are our prompts and skill bundles getting better over time?
Only official benchmark outputs should be committed here.
## What Counts As Official
Tracked benchmark entries should come from one of these sources:
- post-merge runs on `main`
- scheduled nightly benchmark runs
- manually promoted benchmark runs the team wants to preserve
Ad hoc local experiments should stay under `ai_evals/results/` or another
untracked location.
## File Layout
- `benchmark-run.schema.json`: contract for one official run snapshot
- `runs/`: detailed per-run JSON snapshots
- `summary.jsonl`: one compact summary row per official run
## Summary Metrics
Each official run should record enough information to compare quality,
reliability, efficiency, and provenance over time.
The expected metric groups are:
- quality: pass rate and judge-score rollups
- reliability: run-count and flake-related metrics
- efficiency: latency, tokens, tool usage, and cost metrics
- provenance: git SHA, suite/scoring version, provider, and model identity
## Usage
Append one official run snapshot with:
```bash
node ai_evals/scripts/append-official-run.mjs --input /path/to/run.json
```
From the benchmark CLI, the usual path is:
```bash
cd ai_evals
bun run cli -- promote-result ai_evals/results/some-run.json --label main
```
This command will:
1. validate the input shape
2. write the detailed snapshot under `runs/`
3. upsert one summary row in `summary.jsonl`
The writer is intentionally simple. Official history is just the detailed run
files plus the compact summary index.

238
ai_evals/history/benchmark-run.schema.json
View File

@@ -1,238 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://windmill.dev/schemas/ai-evals/benchmark-run.schema.json",
"title": "Official AI Eval Benchmark Run",
"type": "object",
"required": [
"timestamp",
"git_sha",
"suite_version",
"scoring_version",
"surface",
"provider",
"model",
"runs_per_case",
"case_count",
"metrics",
"cases"
],
"properties": {
"run_id": {
"type": "string",
"minLength": 1
},
"timestamp": {
"type": "string",
"format": "date-time"
},
"git_sha": {
"type": "string",
"minLength": 7
},
"suite_version": {
"type": "string",
"minLength": 1
},
"scoring_version": {
"type": "string",
"minLength": 1
},
"surface": {
"type": "string",
"minLength": 1
},
"label": {
"type": "string",
"minLength": 1
},
"provider": {
"type": "string",
"minLength": 1
},
"model": {
"type": "string",
"minLength": 1
},
"judge_model": {
"type": [
"string",
"null"
]
},
"runs_per_case": {
"type": "integer",
"minimum": 1
},
"case_count": {
"type": "integer",
"minimum": 1
},
"metrics": {
"type": "object",
"required": [
"quality",
"reliability",
"efficiency"
],
"properties": {
"quality": {
"type": "object",
"required": [
"pass_rate",
"deterministic_pass_rate",
"judge_score_mean",
"judge_score_median",
"judge_score_p10",
"quality_score"
],
"properties": {
"pass_rate": {
"type": "number",
"minimum": 0,
"maximum": 1
},
"deterministic_pass_rate": {
"type": "number",
"minimum": 0,
"maximum": 1
},
"judge_score_mean": {
"type": "number"
},
"judge_score_median": {
"type": "number"
},
"judge_score_p10": {
"type": "number"
},
"quality_score": {
"type": "number"
},
"category_pass_rate": {
"type": "object",
"additionalProperties": {
"type": "number",
"minimum": 0,
"maximum": 1
}
}
},
"additionalProperties": true
},
"reliability": {
"type": "object",
"required": [
"runs_per_case",
"flake_rate",
"path_consistency"
],
"properties": {
"runs_per_case": {
"type": "integer",
"minimum": 1
},
"flake_rate": {
"type": "number",
"minimum": 0,
"maximum": 1
},
"path_consistency": {
"type": "number",
"minimum": 0,
"maximum": 1
}
},
"additionalProperties": true
},
"efficiency": {
"type": "object",
"required": [
"latency_ms_mean",
"latency_ms_median",
"tokens_total_mean",
"tool_calls_mean",
"iterations_mean",
"estimated_cost_mean",
"cost_per_success",
"latency_per_success",
"efficiency_score",
"value_score"
],
"properties": {
"latency_ms_mean": {
"type": "number",
"minimum": 0
},
"latency_ms_median": {
"type": "number",
"minimum": 0
},
"tokens_prompt_mean": {
"type": "number",
"minimum": 0
},
"tokens_completion_mean": {
"type": "number",
"minimum": 0
},
"tokens_total_mean": {
"type": "number",
"minimum": 0
},
"tool_calls_mean": {
"type": "number",
"minimum": 0
},
"iterations_mean": {
"type": "number",
"minimum": 0
},
"estimated_cost_mean": {
"type": "number",
"minimum": 0
},
"cost_per_success": {
"type": "number",
"minimum": 0
},
"latency_per_success": {
"type": "number",
"minimum": 0
},
"efficiency_score": {
"type": "number"
},
"value_score": {
"type": "number"
}
},
"additionalProperties": true
}
},
"additionalProperties": false
},
"cases": {
"type": "array",
"items": {
"type": "object",
"required": [
"id",
"pass_rate"
],
"properties": {
"id": {
"type": "string",
"minLength": 1
},
"pass_rate": {
"type": "number",
"minimum": 0,
"maximum": 1
}
},
"additionalProperties": true
}
}
},
"additionalProperties": true
}

1
ai_evals/history/runs/.gitkeep
View File

@@ -1 +0,0 @@

184
ai_evals/history/runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json
View File

@@ -1,184 +0,0 @@
{
"timestamp": "2026-04-03T16:13:24.771Z",
"git_sha": "8c9603a01fa48a3e45b3380daac7814b9ed46acf",
"suite_version": "cli-benchmark-v1",
"scoring_version": "cli-deterministic-v1",
"surface": "frontend-flow",
"label": "baseline",
"provider": "anthropic",
"model": "claude-haiku-4-5-20251001",
"judge_model": "claude-sonnet-4-6",
"runs_per_case": 1,
"case_count": 7,
"metrics": {
"quality": {
"pass_rate": 0.8571428571428571,
"deterministic_pass_rate": 0.8571428571428571,
"judge_score_mean": 85.42857142857143,
"judge_score_median": 88,
"judge_score_p10": 72,
"quality_score": 85.71428571428571
},
"reliability": {
"runs_per_case": 1,
"flake_rate": 0,
"path_consistency": 1
},
"efficiency": {
"latency_ms_mean": 143566.42857142858,
"latency_ms_median": 147598,
"tokens_total_mean": 0,
"tool_calls_mean": 11.714285714285714,
"iterations_mean": 12.714285714285714,
"estimated_cost_mean": 0,
"cost_per_success": 0,
"latency_per_success": 141550.33333333334,
"efficiency_score": 34.1017456040819,
"value_score": 29.230067660641627
}
},
"cases": [
{
"id": "flow-test1-user-role-actions",
"pass_rate": 1,
"average_duration_ms": 144460,
"median_duration_ms": 144460,
"latency_per_success_ms": 144460,
"average_assistant_messages": 11,
"average_tool_calls": 10,
"average_skill_invocations": 0,
"path_consistency": 1,
"distinct_skills_invoked": [],
"distinct_tools_used": [
"get_instructions_for_code_generation",
"get_lint_errors",
"set_flow_json",
"test_run_flow"
],
"required_failure_counts": []
},
{
"id": "flow-test2-order-processing",
"pass_rate": 0,
"average_duration_ms": 155663,
"median_duration_ms": 155663,
"latency_per_success_ms": 0,
"average_assistant_messages": 11,
"average_tool_calls": 12,
"average_skill_invocations": 0,
"path_consistency": 1,
"distinct_skills_invoked": [],
"distinct_tools_used": [
"get_instructions_for_code_generation",
"get_lint_errors",
"set_flow_json",
"test_run_flow"
],
"required_failure_counts": [
{
"name": "judge score >= 80",
"count": 1
}
]
},
{
"id": "flow-test3-data-pipeline",
"pass_rate": 1,
"average_duration_ms": 109670,
"median_duration_ms": 109670,
"latency_per_success_ms": 109670,
"average_assistant_messages": 15,
"average_tool_calls": 14,
"average_skill_invocations": 0,
"path_consistency": 1,
"distinct_skills_invoked": [],
"distinct_tools_used": [
"get_instructions_for_code_generation",
"get_lint_errors",
"set_flow_json",
"test_run_flow"
],
"required_failure_counts": []
},
{
"id": "flow-test4-ai-agent-tools",
"pass_rate": 1,
"average_duration_ms": 153811,
"median_duration_ms": 153811,
"latency_per_success_ms": 153811,
"average_assistant_messages": 11,
"average_tool_calls": 7,
"average_skill_invocations": 0,
"path_consistency": 1,
"distinct_skills_invoked": [],
"distinct_tools_used": [
"get_instructions_for_code_generation",
"get_lint_errors",
"set_flow_json",
"test_run_flow"
],
"required_failure_counts": []
},
{
"id": "flow-test5-simple-modification",
"pass_rate": 1,
"average_duration_ms": 143197,
"median_duration_ms": 143197,
"latency_per_success_ms": 143197,
"average_assistant_messages": 15,
"average_tool_calls": 10,
"average_skill_invocations": 0,
"path_consistency": 1,
"distinct_skills_invoked": [],
"distinct_tools_used": [
"get_lint_errors",
"inspect_inline_script",
"set_flow_json",
"set_module_code",
"test_run_flow"
],
"required_failure_counts": []
},
{
"id": "flow-test6-branching-in-loop",
"pass_rate": 1,
"average_duration_ms": 147598,
"median_duration_ms": 147598,
"latency_per_success_ms": 147598,
"average_assistant_messages": 13,
"average_tool_calls": 13,
"average_skill_invocations": 0,
"path_consistency": 1,
"distinct_skills_invoked": [],
"distinct_tools_used": [
"get_instructions_for_code_generation",
"get_lint_errors",
"inspect_inline_script",
"set_flow_json",
"test_run_flow"
],
"required_failure_counts": []
},
{
"id": "flow-test7-parallel-refactor",
"pass_rate": 1,
"average_duration_ms": 150566,
"median_duration_ms": 150566,
"latency_per_success_ms": 150566,
"average_assistant_messages": 13,
"average_tool_calls": 16,
"average_skill_invocations": 0,
"path_consistency": 1,
"distinct_skills_invoked": [],
"distinct_tools_used": [
"get_instructions_for_code_generation",
"get_lint_errors",
"inspect_inline_script",
"set_flow_json",
"test_run_flow"
],
"required_failure_counts": []
}
],
"run_id": "2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4"
}

1
ai_evals/history/summary.jsonl
View File

@@ -1 +0,0 @@
{"run_id":"2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4","timestamp":"2026-04-03T16:13:24.771Z","git_sha":"8c9603a01fa48a3e45b3380daac7814b9ed46acf","suite_version":"cli-benchmark-v1","scoring_version":"cli-deterministic-v1","surface":"frontend-flow","label":"baseline","provider":"anthropic","model":"claude-haiku-4-5-20251001","judge_model":"claude-sonnet-4-6","runs_per_case":1,"case_count":7,"run_path":"runs/2026-04-03t16-13-24-771z__frontend-flow__baseline__anthropic__claude-haiku-4-5-20251001__8c9603a01fa4.json","metrics":{"quality":{"pass_rate":0.8571428571428571,"deterministic_pass_rate":0.8571428571428571,"judge_score_mean":85.42857142857143,"judge_score_median":88,"judge_score_p10":72,"quality_score":85.71428571428571},"reliability":{"runs_per_case":1,"flake_rate":0,"path_consistency":1},"efficiency":{"latency_ms_mean":143566.42857142858,"latency_ms_median":147598,"tokens_total_mean":0,"tool_calls_mean":11.714285714285714,"iterations_mean":12.714285714285714,"estimated_cost_mean":0,"cost_per_success":0,"latency_per_success":141550.33333333334,"efficiency_score":34.1017456040819,"value_score":29.230067660641627}}}

338
ai_evals/history/writer.mjs
View File

@@ -1,338 +0,0 @@
import { mkdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
export const DEFAULT_HISTORY_DIR = MODULE_DIR;
const SUMMARY_FILENAME = "summary.jsonl";
const RUNS_DIRNAME = "runs";
export async function appendOfficialRun(input, options = {}) {
const absoluteHistoryDir = path.resolve(options.historyDir ?? DEFAULT_HISTORY_DIR);
const normalizedRun = normalizeRun(input);
await ensureHistoryLayout(absoluteHistoryDir);
const runFilename = `${normalizedRun.run_id}.json`;
const runRelativePath = path.posix.join(RUNS_DIRNAME, runFilename);
const runFilePath = path.join(absoluteHistoryDir, runRelativePath);
await writeJsonFile(runFilePath, normalizedRun);
const summaryPath = path.join(absoluteHistoryDir, SUMMARY_FILENAME);
const summaries = await loadSummaryEntries(summaryPath);
const summaryEntry = buildSummaryEntry(normalizedRun, runRelativePath);
const nextSummaries = upsertSummaryEntry(summaries, summaryEntry);
await writeSummaryEntries(summaryPath, nextSummaries);
return {
status: "ok",
runId: normalizedRun.run_id,
runPath: runRelativePath,
summaryEntries: nextSummaries.length
};
}
export async function loadSummaryHistory(historyDir = DEFAULT_HISTORY_DIR) {
const summaryPath = path.join(path.resolve(historyDir), SUMMARY_FILENAME);
return await loadSummaryEntries(summaryPath);
}
function normalizeRun(input) {
assertPlainObject(input, "benchmark run");
const timestamp = assertIsoDateTime(input.timestamp, "timestamp");
const gitSha = assertNonEmptyString(input.git_sha, "git_sha");
const suiteVersion = assertNonEmptyString(input.suite_version, "suite_version");
const scoringVersion = assertNonEmptyString(
input.scoring_version,
"scoring_version"
);
const surface = assertNonEmptyString(input.surface, "surface");
const label = assertNonEmptyString(input.label, "label");
const provider = assertNonEmptyString(input.provider, "provider");
const model = assertNonEmptyString(input.model, "model");
const judgeModel =
input.judge_model === null || input.judge_model === undefined
? null
: assertNonEmptyString(input.judge_model, "judge_model");
const runsPerCase = assertPositiveInteger(input.runs_per_case, "runs_per_case");
const caseCount = assertPositiveInteger(input.case_count, "case_count");
const metrics = normalizeMetrics(input.metrics, runsPerCase);
const cases = normalizeCases(input.cases);
if (cases.length !== caseCount) {
throw new Error(
`case_count (${caseCount}) does not match cases.length (${cases.length})`
);
}
const runId =
input.run_id && typeof input.run_id === "string" && input.run_id.trim()
? input.run_id.trim()
: buildRunId({
timestamp,
surface,
label,
provider,
model,
gitSha
});
return {
...input,
run_id: runId,
timestamp,
git_sha: gitSha,
suite_version: suiteVersion,
scoring_version: scoringVersion,
surface,
label,
provider,
model,
judge_model: judgeModel,
runs_per_case: runsPerCase,
case_count: caseCount,
metrics,
cases
};
}
function normalizeMetrics(input, runsPerCase) {
assertPlainObject(input, "metrics");
const quality = normalizeMetricGroup(
input.quality,
"metrics.quality",
[
"pass_rate",
"deterministic_pass_rate",
"judge_score_mean",
"judge_score_median",
"judge_score_p10",
"quality_score"
],
new Set(["pass_rate", "deterministic_pass_rate"])
);
const reliability = normalizeMetricGroup(
input.reliability,
"metrics.reliability",
["runs_per_case", "flake_rate", "path_consistency"],
new Set(["flake_rate", "path_consistency"])
);
const efficiency = normalizeMetricGroup(
input.efficiency,
"metrics.efficiency",
[
"latency_ms_mean",
"latency_ms_median",
"tokens_total_mean",
"tool_calls_mean",
"iterations_mean",
"estimated_cost_mean",
"cost_per_success",
"latency_per_success",
"efficiency_score",
"value_score"
]
);
if (reliability.runs_per_case !== runsPerCase) {
throw new Error(
`metrics.reliability.runs_per_case (${reliability.runs_per_case}) does not match runs_per_case (${runsPerCase})`
);
}
if (quality.category_pass_rate !== undefined) {
assertPlainObject(quality.category_pass_rate, "metrics.quality.category_pass_rate");
for (const [category, value] of Object.entries(quality.category_pass_rate)) {
assertRatio(value, `metrics.quality.category_pass_rate.${category}`);
}
}
return { quality, reliability, efficiency };
}
function normalizeMetricGroup(input, groupName, requiredFields, ratioFields = new Set()) {
assertPlainObject(input, groupName);
const normalized = { ...input };
for (const field of requiredFields) {
if (!(field in normalized)) {
throw new Error(`Missing required ${groupName}.${field}`);
}
if (field === "runs_per_case") {
normalized[field] = assertPositiveInteger(
normalized[field],
`${groupName}.${field}`
);
continue;
}
normalized[field] = assertFiniteNumber(normalized[field], `${groupName}.${field}`);
if (ratioFields.has(field)) {
assertRatio(normalized[field], `${groupName}.${field}`);
}
}
for (const [field, value] of Object.entries(normalized)) {
if (value === null || value === undefined) {
continue;
}
if (typeof value === "number") {
normalized[field] = assertFiniteNumber(value, `${groupName}.${field}`);
if (ratioFields.has(field)) {
assertRatio(normalized[field], `${groupName}.${field}`);
}
}
}
return normalized;
}
function normalizeCases(input) {
if (!Array.isArray(input) || input.length === 0) {
throw new Error("cases must be a non-empty array");
}
return input.map((entry, index) => {
assertPlainObject(entry, `cases[${index}]`);
return {
...entry,
id: assertNonEmptyString(entry.id, `cases[${index}].id`),
pass_rate: assertRatio(entry.pass_rate, `cases[${index}].pass_rate`)
};
});
}
function buildRunId({ timestamp, surface, label, provider, model, gitSha }) {
const timestampSlug = timestamp.replaceAll(":", "-").replaceAll(".", "-");
const shortSha = gitSha.slice(0, 12);
return [
slugify(timestampSlug),
slugify(surface),
slugify(label),
slugify(provider),
slugify(model),
shortSha
]
.filter(Boolean)
.join("__");
}
function buildSummaryEntry(run, runRelativePath) {
return {
run_id: run.run_id,
timestamp: run.timestamp,
git_sha: run.git_sha,
suite_version: run.suite_version,
scoring_version: run.scoring_version,
surface: run.surface,
label: run.label,
provider: run.provider,
model: run.model,
judge_model: run.judge_model,
runs_per_case: run.runs_per_case,
case_count: run.case_count,
run_path: runRelativePath,
metrics: run.metrics
};
}
function upsertSummaryEntry(entries, nextEntry) {
const remainingEntries = entries.filter((entry) => entry.run_id !== nextEntry.run_id);
const nextEntries = [...remainingEntries, nextEntry];
nextEntries.sort((left, right) => left.timestamp.localeCompare(right.timestamp));
return nextEntries;
}
async function loadSummaryEntries(summaryPath) {
try {
const raw = await readFile(summaryPath, "utf8");
return raw
.split("\n")
.map((line) => line.trim())
.filter(Boolean)
.map((line, index) => {
try {
return JSON.parse(line);
} catch (error) {
throw new Error(
`Failed to parse ${SUMMARY_FILENAME} line ${index + 1}: ${error.message}`
);
}
});
} catch (error) {
if (error.code === "ENOENT") {
return [];
}
throw error;
}
}
async function writeSummaryEntries(summaryPath, entries) {
const content =
entries.map((entry) => JSON.stringify(entry)).join("\n") +
(entries.length > 0 ? "\n" : "");
await writeFile(summaryPath, content, "utf8");
}
async function ensureHistoryLayout(historyDir) {
await mkdir(path.join(historyDir, RUNS_DIRNAME), { recursive: true });
}
async function writeJsonFile(filePath, value) {
await writeFile(filePath, JSON.stringify(value, null, 2) + "\n", "utf8");
}
function assertPlainObject(value, fieldName) {
if (!value || typeof value !== "object" || Array.isArray(value)) {
throw new Error(`${fieldName} must be an object`);
}
}
function assertNonEmptyString(value, fieldName) {
if (typeof value !== "string" || value.trim().length === 0) {
throw new Error(`${fieldName} must be a non-empty string`);
}
return value.trim();
}
function assertPositiveInteger(value, fieldName) {
if (!Number.isInteger(value) || value < 1) {
throw new Error(`${fieldName} must be a positive integer`);
}
return value;
}
function assertFiniteNumber(value, fieldName) {
if (typeof value !== "number" || !Number.isFinite(value)) {
throw new Error(`${fieldName} must be a finite number`);
}
return value;
}
function assertRatio(value, fieldName) {
const normalized = assertFiniteNumber(value, fieldName);
if (normalized < 0 || normalized > 1) {
throw new Error(`${fieldName} must be between 0 and 1`);
}
return normalized;
}
function assertIsoDateTime(value, fieldName) {
const normalized = assertNonEmptyString(value, fieldName);
if (Number.isNaN(Date.parse(normalized))) {
throw new Error(`${fieldName} must be a valid ISO date-time string`);
}
return normalized;
}
function slugify(value) {
return value
.toLowerCase()
.replaceAll(/[^a-z0-9]+/g, "-")
.replaceAll(/^-+|-+$/g, "");
}

41
ai_evals/modes/app.ts Normal file
View File

@@ -0,0 +1,41 @@
import { loadAppFixture } from "../adapters/frontend/core/app/appFixtureLoader";
import type { AppFiles } from "../../frontend/src/lib/components/copilot/chat/app/core";
import { validateAppState, type AppFilesState } from "../core/validators";
import type { ModeRunner } from "../core/types";
import { runAppEval } from "../adapters/frontend/core/app/appEvalRunner";
import { FRONTEND_MODEL, FRONTEND_PROVIDER, getFrontendApiKey } from "./frontendCommon";
export function createAppModeRunner(): ModeRunner<AppFilesState, AppFilesState, AppFilesState> {
return {
mode: "app",
concurrency: 5,
judgeThreshold: 80,
async loadInitial(path) {
return path ? (await loadAppFixture(path)) : undefined;
},
async loadExpected(path) {
return path ? (await loadAppFixture(path)) : undefined;
},
async run(prompt, initial) {
const result = await runAppEval(prompt, getFrontendApiKey(), {
initialFrontend: initial?.frontend,
initialBackend: initial?.backend as AppFiles["backend"] | undefined,
provider: FRONTEND_PROVIDER,
model: FRONTEND_MODEL,
});
return {
success: result.success,
actual: result.files as AppFilesState,
error: result.error,
assistantMessageCount: result.assistantMessageCount,
toolCallCount: result.toolCallCount,
toolsUsed: result.toolsUsed,
skillsInvoked: [],
};
},
validate({ actual, initial, expected }) {
return validateAppState({ actual, initial, expected });
},
};
}

129
ai_evals/modes/cli.ts Normal file
View File

@@ -0,0 +1,129 @@
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { dirname, join } from "node:path";
import { readFile } from "node:fs/promises";
import { writeAiGuidanceFiles } from "../../cli/src/guidance/writer.ts";
import { getGeneratedSkillsSource, runPromptAndCapture, CLI_BENCHMARK_MODEL, CLI_BENCHMARK_PROVIDER } from "../adapters/cli/runtime";
import { copyDirectory, readDirectoryFiles } from "../core/files";
import { validateCliWorkspace } from "../core/validators";
import type { ModeRunner } from "../core/types";
const IGNORE_WORKSPACE_FILES = new Set([".claude", "AGENTS.md", "CLAUDE.md", "rt.d.ts"]);
interface CliWorkspaceFixture {
sourceDir: string;
files: Record<string, string>;
}
interface CliRunActual {
assistantOutput: string;
workspaceFiles: Record<string, string>;
}
const CLAUDE_PROJECT_PREAMBLE = [
"Follow the project instructions from AGENTS.md exactly.",
"Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.",
"Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
"Do not skip the Skill step.",
].join(" ");
export function createCliModeRunner(): ModeRunner<CliWorkspaceFixture, CliWorkspaceFixture, CliRunActual> {
return {
mode: "cli",
concurrency: 1,
judgeThreshold: 80,
async loadInitial(path) {
return path
? {
sourceDir: path,
files: await readDirectoryFiles(path),
}
: undefined;
},
async loadExpected(path) {
return path
? {
sourceDir: path,
files: await readDirectoryFiles(path),
}
: undefined;
},
async run(prompt, initial) {
const workspaceDir = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-"));
try {
if (initial) {
await copyDirectory(initial.sourceDir, workspaceDir);
}
await mkdir(dirname(join(workspaceDir, ".claude", "skills")), { recursive: true });
await writeAiGuidanceFiles({
targetDir: workspaceDir,
nonDottedPaths: true,
overwriteProjectGuidance: true,
skillsSourcePath: getGeneratedSkillsSource(),
});
await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8");
const renderedPrompt = await renderPrompt(prompt, workspaceDir);
const run = await runPromptAndCapture(renderedPrompt, workspaceDir, 6);
const workspaceFiles = await readDirectoryFiles(workspaceDir, { ignore: IGNORE_WORKSPACE_FILES });
return {
success: true,
actual: {
assistantOutput: run.output,
workspaceFiles,
},
assistantMessageCount: run.assistantMessageCount,
toolCallCount: run.toolsUsed.length,
toolsUsed: run.toolsUsed.map((entry) => entry.tool),
skillsInvoked: run.skillsInvoked,
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return {
success: false,
actual: {
assistantOutput: "",
workspaceFiles: {},
},
error: message,
assistantMessageCount: 0,
toolCallCount: 0,
toolsUsed: [],
skillsInvoked: [],
};
} finally {
await rm(workspaceDir, { recursive: true, force: true });
}
},
validate({ actual, initial, expected }) {
return validateCliWorkspace({
actualFiles: actual.workspaceFiles,
expectedFiles: expected?.files,
initialFiles: initial?.files,
});
},
};
}
export function getCliRunModelLabel(): string {
return `${CLI_BENCHMARK_PROVIDER}:${CLI_BENCHMARK_MODEL}`;
}
async function renderPrompt(prompt: string, workspaceDir: string): Promise<string> {
const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir);
const agentsInstructions = await readFile(path.join(workspaceDir, "AGENTS.md"), "utf8");
return [
"# Project Instructions",
agentsInstructions.trim(),
"",
"# Benchmark Harness",
CLAUDE_PROJECT_PREAMBLE,
"",
"# User Request",
renderedUserPrompt,
].join("\n");
}

42
ai_evals/modes/flow.ts Normal file
View File

@@ -0,0 +1,42 @@
import { readJsonFile } from "../core/files";
import { validateFlowState, type FlowState } from "../core/validators";
import type { ModeRunner } from "../core/types";
import { runFlowEval } from "../adapters/frontend/core/flow/flowEvalRunner";
import { FRONTEND_MODEL, FRONTEND_PROVIDER, getFrontendApiKey } from "./frontendCommon";
export function createFlowModeRunner(): ModeRunner<FlowState, FlowState, FlowState> {
return {
mode: "flow",
concurrency: 5,
judgeThreshold: 80,
async loadInitial(path) {
return path ? await readJsonFile<FlowState>(path) : undefined;
},
async loadExpected(path) {
return path ? await readJsonFile<FlowState>(path) : undefined;
},
async run(prompt, initial) {
const result = await runFlowEval(prompt, getFrontendApiKey(), {
initialFlow: initial,
provider: FRONTEND_PROVIDER,
model: FRONTEND_MODEL,
});
return {
success: result.success,
actual: {
value: { modules: result.flow.value?.modules ?? [] },
schema: result.flow.schema,
},
error: result.error,
assistantMessageCount: result.assistantMessageCount,
toolCallCount: result.toolCallCount,
toolsUsed: result.toolsUsed,
skillsInvoked: [],
};
},
validate({ actual, expected }) {
return validateFlowState({ actual, expected });
},
};
}

14
ai_evals/modes/frontendCommon.ts Normal file
View File

@@ -0,0 +1,14 @@
export const FRONTEND_PROVIDER = "anthropic";
export const FRONTEND_MODEL = "claude-haiku-4-5-20251001";
export function getFrontendApiKey(): string {
const apiKey = process.env.ANTHROPIC_API_KEY;
if (!apiKey) {
throw new Error("ANTHROPIC_API_KEY is required for frontend evals");
}
return apiKey;
}
export function getFrontendRunModelLabel(): string {
return `${FRONTEND_PROVIDER}:${FRONTEND_MODEL}`;
}

44
ai_evals/modes/script.ts Normal file
View File

@@ -0,0 +1,44 @@
import { readJsonFile } from "../core/files";
import { validateScriptState } from "../core/validators";
import type { ModeRunner } from "../core/types";
import { runScriptEval } from "../adapters/frontend/core/script/scriptEvalRunner";
import type { ScriptEvalState } from "../adapters/frontend/core/script/fileHelpers";
import { FRONTEND_MODEL, FRONTEND_PROVIDER, getFrontendApiKey } from "./frontendCommon";
export function createScriptModeRunner(): ModeRunner<ScriptEvalState, ScriptEvalState, ScriptEvalState> {
return {
mode: "script",
concurrency: 5,
judgeThreshold: 80,
async loadInitial(path) {
return path ? await readJsonFile<ScriptEvalState>(path) : undefined;
},
async loadExpected(path) {
return path ? await readJsonFile<ScriptEvalState>(path) : undefined;
},
async run(prompt, initial) {
if (!initial) {
throw new Error("Script evals require an initial script fixture");
}
const result = await runScriptEval(prompt, getFrontendApiKey(), {
initialScript: initial,
provider: FRONTEND_PROVIDER,
model: FRONTEND_MODEL,
});
return {
success: result.success,
actual: result.script,
error: result.error,
assistantMessageCount: result.assistantMessageCount,
toolCallCount: result.toolCallCount,
toolsUsed: result.toolsUsed,
skillsInvoked: [],
};
},
validate({ actual, initial, expected }) {
return validateScriptState({ actual, initial, expected });
},
};
}

4
ai_evals/package.json
View File

@@ -3,10 +3,10 @@
"private": true,
"type": "module",
"scripts": {
"cli": "bun cli/index.ts",
"list-cases": "bun cli/index.ts list-cases"
"cli": "bun cli/index.ts"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.39.0",
"@anthropic-ai/claude-agent-sdk": "^0.2.25",
"commander": "^14.0.3"
},

63
ai_evals/scripts/append-official-run.mjs
View File

@@ -1,63 +0,0 @@
#!/usr/bin/env node
import { readFile } from "node:fs/promises";
import path from "node:path";
import { appendOfficialRun, DEFAULT_HISTORY_DIR } from "../history/writer.mjs";
async function main() {
const { inputPath, historyDir } = parseArgs(process.argv.slice(2));
const input = JSON.parse(await readFile(path.resolve(inputPath), "utf8"));
const result = await appendOfficialRun(input, {
historyDir
});
process.stdout.write(JSON.stringify(result, null, 2) + "\n");
}
function parseArgs(argv) {
let inputPath;
let historyDir = DEFAULT_HISTORY_DIR;
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--input") {
inputPath = argv[index + 1];
index += 1;
continue;
}
if (arg === "--history-dir") {
historyDir = argv[index + 1];
index += 1;
continue;
}
if (arg === "--help" || arg === "-h") {
printHelp();
process.exit(0);
}
throw new Error(`Unknown argument: ${arg}`);
}
if (!inputPath) {
throw new Error("Missing required --input /path/to/run.json argument");
}
return { inputPath, historyDir };
}
function printHelp() {
process.stdout.write(
[
"Usage:",
" node ai_evals/scripts/append-official-run.mjs --input /path/to/run.json",
"",
"Options:",
" --history-dir /path/to/history Override the history directory",
" --help Show this message"
].join("\n") + "\n"
);
}
main().catch((error) => {
process.stderr.write(`${error.message}\n`);
process.exit(1);
});

34
frontend/src/lib/components/copilot/chat/shared.ts
View File

@@ -708,6 +708,10 @@ export async function pollJobCompletion(
toolId: string,
toolCallbacks: ToolCallbacks
): Promise<CompletedJob> {
if (isBenchmarkMockJob(jobId, workspace)) {
return buildBenchmarkMockCompletedJob(jobId, workspace)
}
let attempts = 0
const maxAttempts = 60
let job: CompletedJob | null = null
@@ -746,6 +750,36 @@ export async function pollJobCompletion(
return job
}
function isBenchmarkMockJob(jobId: string, workspace: string): boolean {
return jobId.startsWith('mock-job-id-') && workspace.includes('wmill-frontend-')
}
function buildBenchmarkMockCompletedJob(jobId: string, workspace: string): CompletedJob {
const now = new Date().toISOString()
return {
id: jobId,
created_by: 'ai-evals',
created_at: now,
started_at: now,
completed_at: now,
duration_ms: 0,
success: true,
result: {
mocked: true,
workspace
},
logs: 'Mock benchmark test run completed successfully.',
canceled: false,
job_kind: 'flowpreview',
permissioned_as: 'u/ai-evals',
is_flow_step: false,
is_skipped: false,
email: 'ai-evals@local',
visible_to_owner: true,
tag: 'benchmark'
}
}
// Helper function to extract code blocks from markdown text
export function extractCodeFromMarkdown(markdown: string): string[] {
const codeBlocks: string[] = []