From 9bd0dc8d1fd4d567134b6d080291b91092e983ea Mon Sep 17 00:00:00 2001 From: centdix Date: Wed, 8 Apr 2026 18:06:55 +0200 Subject: [PATCH] refactor: simplify and harden ai eval benchmarks Co-Authored-By: Claude Opus 4.5 --- ai_evals/README.md | 7 +- ai_evals/adapters/cli/runtime.ts | 13 +- ai_evals/adapters/frontend/benchmarkRunner.ts | 25 +- ai_evals/adapters/frontend/runtime.ts | 2 + ai_evals/bun.lock | 3 + ai_evals/cases/app.json | 44 -- ai_evals/cases/app.yaml | 46 ++ ai_evals/cases/cli.json | 12 - ai_evals/cases/cli.yaml | 21 + ai_evals/cases/flow.json | 72 --- ai_evals/cases/flow.yaml | 186 +++++++ ai_evals/cases/script.json | 8 - ai_evals/cases/script.yaml | 7 + ai_evals/cli/index.ts | 70 ++- ai_evals/core/cases.ts | 9 +- ai_evals/core/judge.ts | 4 +- ai_evals/core/models.ts | 149 ++++++ ai_evals/core/results.ts | 78 ++- ai_evals/core/runSuite.ts | 4 + ai_evals/core/types.ts | 29 ++ ai_evals/core/validators.ts | 465 +++++++++++++++++- ai_evals/modes/app.ts | 48 +- ai_evals/modes/cli.ts | 39 +- ai_evals/modes/flow.ts | 32 +- ai_evals/modes/frontendCommon.ts | 23 +- ai_evals/modes/script.ts | 27 +- ai_evals/package.json | 5 +- 27 files changed, 1223 insertions(+), 205 deletions(-) delete mode 100644 ai_evals/cases/app.json create mode 100644 ai_evals/cases/app.yaml delete mode 100644 ai_evals/cases/cli.json create mode 100644 ai_evals/cases/cli.yaml delete mode 100644 ai_evals/cases/flow.json create mode 100644 ai_evals/cases/flow.yaml delete mode 100644 ai_evals/cases/script.json create mode 100644 ai_evals/cases/script.yaml create mode 100644 ai_evals/core/models.ts diff --git a/ai_evals/README.md b/ai_evals/README.md index 7b081ab123..d5bfca4789 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -59,9 +59,14 @@ bun run cli -- run cli bun-hello-script `run` always writes a JSON result file under `ai_evals/results/` unless you pass `--output`. +It also writes generated artifacts next to that summary file, for example: + +- summary: `ai_evals/results/2026-04-08T13-00-00.000Z__flow.json` +- artifacts: `ai_evals/results/2026-04-08T13-00-00.000Z__flow//attempt-1/flow.json` + ## Layout -- `cases/`: one JSON file per mode +- `cases/`: one YAML file per mode - `fixtures/`: initial and expected fixtures - `core/`: shared case loading, validation, judging, and result writing - `modes/`: one runner per mode diff --git a/ai_evals/adapters/cli/runtime.ts b/ai_evals/adapters/cli/runtime.ts index ba144a305b..a4a31a7ef9 100644 --- a/ai_evals/adapters/cli/runtime.ts +++ b/ai_evals/adapters/cli/runtime.ts @@ -1,6 +1,7 @@ import { query, type Options } from "@anthropic-ai/claude-agent-sdk"; import { join } from "path"; import { fileURLToPath } from "url"; +import { getCliEvalModel, resolveEvalModel, type CliEvalModelConfig } from "../../core/models"; export interface ToolInvocation { tool: string; @@ -17,8 +18,7 @@ export interface PromptRunResult { } const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url)); -export const CLI_BENCHMARK_PROVIDER = "anthropic"; -export const CLI_BENCHMARK_MODEL = "haiku"; +export const DEFAULT_CLI_EVAL_MODEL: CliEvalModelConfig = getCliEvalModel(resolveEvalModel("cli")); export function getGeneratedSkillsSource(): string { return join(REPO_ROOT, "system_prompts", "auto-generated", "skills"); @@ -27,7 +27,8 @@ export function getGeneratedSkillsSource(): string { export async function runPromptAndCapture( prompt: string, cwd: string, - maxTurns: number = 3 + maxTurns: number = 3, + modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL ): Promise { const toolsUsed: ToolInvocation[] = []; const skillsInvoked: string[] = []; @@ -37,7 +38,7 @@ export async function runPromptAndCapture( const options: Options = { cwd, - model: CLI_BENCHMARK_MODEL, + model: modelConfig.model, maxTurns, settingSources: ["project"], allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"] @@ -92,6 +93,10 @@ export function wasToolUsed(result: PromptRunResult, toolName: string): boolean return result.toolsUsed.some((tool) => tool.tool === toolName); } +export function formatCliRunModelLabel(modelConfig: CliEvalModelConfig): string { + return `${modelConfig.provider}:${modelConfig.model}`; +} + export function getToolInputs( result: PromptRunResult, toolName: string diff --git a/ai_evals/adapters/frontend/benchmarkRunner.ts b/ai_evals/adapters/frontend/benchmarkRunner.ts index a2f8556173..33b1555654 100644 --- a/ai_evals/adapters/frontend/benchmarkRunner.ts +++ b/ai_evals/adapters/frontend/benchmarkRunner.ts @@ -1,4 +1,9 @@ import { loadSelectedCases } from "../../core/cases"; +import { + formatRunModelLabel, + getFrontendEvalModel, + resolveEvalModel, +} from "../../core/models"; import { buildRunResult } from "../../core/results"; import { runSuite } from "../../core/runSuite"; import type { BenchmarkRunResult, ModeRunner } from "../../core/types"; @@ -7,7 +12,6 @@ import { createAppModeRunner } from "../../modes/app"; import { createFlowModeRunner } from "../../modes/flow"; import { createScriptModeRunner } from "../../modes/script"; import { DEFAULT_JUDGE_MODEL } from "../../core/judge"; -import { getFrontendRunModelLabel } from "../../modes/frontendCommon"; export type FrontendBenchmarkMode = "flow" | "app" | "script"; @@ -17,14 +21,16 @@ export async function runFrontendBenchmarkFromEnv(): Promise const runs = parsePositiveInteger(process.env.WMILL_FRONTEND_AI_EVAL_RUNS, "WMILL_FRONTEND_AI_EVAL_RUNS"); const emitProgress = process.env.WMILL_FRONTEND_AI_EVAL_PROGRESS === "1"; const verbose = process.env.WMILL_FRONTEND_AI_EVAL_VERBOSE === "1"; + const model = resolveEvalModel(mode, process.env.WMILL_FRONTEND_AI_EVAL_MODEL); const selectedCases = await loadSelectedCases(mode, caseIds); - const modeRunner = getModeRunner(mode); + const modeRunner = getModeRunner(mode, getFrontendEvalModel(model)); + const runModel = formatRunModelLabel(mode, model); const caseResults = await runSuite({ modeRunner, cases: selectedCases, runs, - runModel: getFrontendRunModelLabel(), + runModel, judgeModel: DEFAULT_JUDGE_MODEL, concurrency: verbose ? 1 : undefined, verbose, @@ -34,20 +40,23 @@ export async function runFrontendBenchmarkFromEnv(): Promise return buildRunResult({ mode, runs, - runModel: getFrontendRunModelLabel(), + runModel, judgeModel: DEFAULT_JUDGE_MODEL, caseResults, }); } -function getModeRunner(mode: FrontendBenchmarkMode): ModeRunner { +function getModeRunner( + mode: FrontendBenchmarkMode, + model: ReturnType +): ModeRunner { switch (mode) { case "flow": - return createFlowModeRunner(); + return createFlowModeRunner(model); case "app": - return createAppModeRunner(); + return createAppModeRunner(model); case "script": - return createScriptModeRunner(); + return createScriptModeRunner(model); } } diff --git a/ai_evals/adapters/frontend/runtime.ts b/ai_evals/adapters/frontend/runtime.ts index d0282b36ae..4d999b858e 100644 --- a/ai_evals/adapters/frontend/runtime.ts +++ b/ai_evals/adapters/frontend/runtime.ts @@ -19,6 +19,7 @@ export async function runFrontendBenchmarkAdapter(input: { mode: FrontendMode caseIds: string[] runs: number + model?: string verbose?: boolean }): Promise { const tempDir = await mkdtemp(path.join(tmpdir(), 'wmill-frontend-benchmark-')) @@ -44,6 +45,7 @@ export async function runFrontendBenchmarkAdapter(input: { WMILL_FRONTEND_AI_EVAL_MODE: input.mode, WMILL_FRONTEND_AI_EVAL_CASE_IDS: JSON.stringify(input.caseIds), WMILL_FRONTEND_AI_EVAL_RUNS: String(input.runs), + WMILL_FRONTEND_AI_EVAL_MODEL: input.model ?? "", WMILL_FRONTEND_AI_EVAL_PROGRESS: '1', WMILL_FRONTEND_AI_EVAL_VERBOSE: input.verbose ? '1' : '0' } diff --git a/ai_evals/bun.lock b/ai_evals/bun.lock index 1f03e89706..da6a96d97e 100644 --- a/ai_evals/bun.lock +++ b/ai_evals/bun.lock @@ -8,6 +8,7 @@ "@anthropic-ai/claude-agent-sdk": "^0.2.25", "@anthropic-ai/sdk": "^0.39.0", "commander": "^14.0.3", + "yaml": "^2.8.3", }, "devDependencies": { "@types/bun": "latest", @@ -286,6 +287,8 @@ "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "yaml": ["yaml@2.8.3", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg=="], + "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="], diff --git a/ai_evals/cases/app.json b/ai_evals/cases/app.json deleted file mode 100644 index 355a376df6..0000000000 --- a/ai_evals/cases/app.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "id": "app-test1-counter-create", - "prompt": "Create a counter app with increment/decrement buttons" - }, - { - "id": "app-test2-counter-reset", - "prompt": "Add a reset button that sets the counter back to 0", - "initial": "ai_evals/fixtures/frontend/app/initial/test1_counter_app" - }, - { - "id": "app-test3-shopping-cart-quantity", - "prompt": "Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items", - "initial": "ai_evals/fixtures/frontend/app/initial/shopping_cart" - }, - { - "id": "app-test4-shopping-cart-discount", - "prompt": "Add a discount code input field in the cart. When the code \"SAVE10\" is entered, apply a 10% discount to the total", - "initial": "ai_evals/fixtures/frontend/app/initial/shopping_cart" - }, - { - "id": "app-test5-file-manager-search", - "prompt": "Add a search bar in the toolbar that filters files and folders by name as the user types", - "initial": "ai_evals/fixtures/frontend/app/initial/file_manager" - }, - { - "id": "app-test6-file-manager-details", - "prompt": "Show file size (formatted as KB/MB) and modified date in the file list for each item", - "initial": "ai_evals/fixtures/frontend/app/initial/file_manager" - }, - { - "id": "app-test7-file-manager-select-all", - "prompt": "Add a \"Select All\" checkbox in the file list header and individual checkboxes for each file. Add a \"Delete Selected\" button that appears when items are selected", - "initial": "ai_evals/fixtures/frontend/app/initial/file_manager" - }, - { - "id": "app-test8-quiz-create", - "prompt": "Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct." - }, - { - "id": "app-test9-recipe-book-create", - "prompt": "Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes." - } -] diff --git a/ai_evals/cases/app.yaml b/ai_evals/cases/app.yaml new file mode 100644 index 0000000000..21955a2ced --- /dev/null +++ b/ai_evals/cases/app.yaml @@ -0,0 +1,46 @@ +- id: app-test1-counter-create + prompt: |- + Create a counter app with increment/decrement buttons + +- id: app-test2-counter-reset + prompt: |- + Add a reset button that sets the counter back to 0 + initial: ai_evals/fixtures/frontend/app/initial/test1_counter_app + +- id: app-test3-shopping-cart-quantity + prompt: |- + Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items + initial: ai_evals/fixtures/frontend/app/initial/shopping_cart + +- id: app-test4-shopping-cart-discount + prompt: |- + Add a discount code input field in the cart. + When the code "SAVE10" is entered, apply a 10% discount to the total + initial: ai_evals/fixtures/frontend/app/initial/shopping_cart + +- id: app-test5-file-manager-search + prompt: |- + Add a search bar in the toolbar that filters files and folders by name as the user types + initial: ai_evals/fixtures/frontend/app/initial/file_manager + +- id: app-test6-file-manager-details + prompt: |- + Show file size (formatted as KB/MB) and modified date in the file list for each item + initial: ai_evals/fixtures/frontend/app/initial/file_manager + +- id: app-test7-file-manager-select-all + prompt: |- + Add a "Select All" checkbox in the file list header and individual checkboxes for each file. + Add a "Delete Selected" button that appears when items are selected + initial: ai_evals/fixtures/frontend/app/initial/file_manager + +- id: app-test8-quiz-create + prompt: |- + Create a multiple choice quiz app with 5 questions about general knowledge. + Show one question at a time with 4 answer options. + Track the score and show results at the end with percentage correct. + +- id: app-test9-recipe-book-create + prompt: |- + Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. + Include a search bar to filter recipes by name and the ability to delete recipes. diff --git a/ai_evals/cases/cli.json b/ai_evals/cases/cli.json deleted file mode 100644 index 6d2eb9757c..0000000000 --- a/ai_evals/cases/cli.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "id": "bun-hello-script", - "prompt": "This is a benchmark harness. Create exactly one Windmill Bun/TypeScript script at {{workspace_root}}/f/evals/hello.ts. The script must export async function main(name: string) and return an object { greeting: `Hello, ${name}!` }. Keep it minimal. Do not create other scripts. Do not run any CLI commands. After writing the file, tell me exactly which wmill commands I should run next.", - "expected": "ai_evals/fixtures/cli/expected/bun-hello-script" - }, - { - "id": "bun-hello-flow", - "prompt": "This is a benchmark harness. Create exactly one Windmill flow folder at {{workspace_root}}/f/evals/hello__flow. The flow must contain flow.yaml and one inline Bun script file named hello.ts. The flow should accept a name string input and return an object { greeting: `Hello, ${name}!` }. Use a single rawscript step wired to that input. Keep it minimal. Do not create any other flows or scripts. Do not run any CLI commands. After writing the files, tell me exactly which wmill commands I should run next.", - "expected": "ai_evals/fixtures/cli/expected/bun-hello-flow" - } -] diff --git a/ai_evals/cases/cli.yaml b/ai_evals/cases/cli.yaml new file mode 100644 index 0000000000..5054f03e0e --- /dev/null +++ b/ai_evals/cases/cli.yaml @@ -0,0 +1,21 @@ +- id: bun-hello-script + prompt: |- + This is a benchmark harness. Create exactly one Windmill Bun/TypeScript script at {{workspace_root}}/f/evals/hello.ts. + The script must export async function main(name: string) and return an object { greeting: `Hello, ${name}!` }. + Keep it minimal. + Do not create other scripts. + Do not run any CLI commands. + After writing the file, tell me exactly which wmill commands I should run next. + expected: ai_evals/fixtures/cli/expected/bun-hello-script + +- id: bun-hello-flow + prompt: |- + This is a benchmark harness. Create exactly one Windmill flow folder at {{workspace_root}}/f/evals/hello__flow. + The flow must contain flow.yaml and one inline Bun script file named hello.ts. + The flow should accept a name string input and return an object { greeting: `Hello, ${name}!` }. + Use a single rawscript step wired to that input. + Keep it minimal. + Do not create any other flows or scripts. + Do not run any CLI commands. + After writing the files, tell me exactly which wmill commands I should run next. + expected: ai_evals/fixtures/cli/expected/bun-hello-flow diff --git a/ai_evals/cases/flow.json b/ai_evals/cases/flow.json deleted file mode 100644 index 96ae17b985..0000000000 --- a/ai_evals/cases/flow.json +++ /dev/null @@ -1,72 +0,0 @@ -[ - { - "id": "flow-test0-sum-two-numbers", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nCreate a flow with a single Bun rawscript step named \"sum_numbers\".\nThe flow input must be two numbers named a and b.\nThe rawscript must read a and b from flow input and return a + b.\nDo not add extra steps, branches, loops, AI agents, or test steps.", - "expected": "ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json" - }, - { - "id": "flow-test1-reuse-existing-script", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nThere is already a workspace script that adds two numbers.\nSearch the workspace for it, inspect it, and reuse it instead of writing inline code.\nCreate a flow with a single script step named \"sum_numbers\".\nThe flow input must be two numbers named a and b.\nThe step must reference the existing script by path and pass a and b from flow input.\nDo not use a rawscript step.\nDo not create or modify other workspace items.\nDo not run tests.", - "initial": "ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json", - "expected": "ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json" - }, - { - "id": "flow-test2-call-existing-subflow", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nThere is already a workspace flow that adds two numbers.\nSearch the workspace for it, inspect it, and create a parent flow that calls it by path.\nUse a single flow step named \"call_add_numbers\".\nThe parent flow input must be two numbers named a and b.\nPass a and b from the parent flow input into the referenced subflow.\nDo not inline code.\nDo not create or modify other workspace items.\nDo not run tests.", - "initial": "ai_evals/fixtures/frontend/flow/initial/test2_call_existing_subflow_initial.json", - "expected": "ai_evals/fixtures/frontend/flow/expected/test2_call_existing_subflow.json" - }, - { - "id": "flow-test3-branchone-routing", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nCreate a flow that routes incoming support requests by tier.\nThe flow input must contain a string field named tier.\nAdd a branchone step named \"route_by_tier\" with explicit branches for \"free\", \"pro\", and \"enterprise\".\nEach branch should return a simple object with the chosen queue name.\nAlso include a default branch for unknown tiers.\nKeep the flow minimal.", - "expected": "ai_evals/fixtures/frontend/flow/expected/test3_branchone_routing.json" - }, - { - "id": "flow-test4-order-processing-loop", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nSTEP 1: Receive order data from input (order has items array with name/price/quantity, customer_email, shipping_address)\nSTEP 2: Validate order - check all items have valid price > 0 and quantity > 0, return validation result\nSTEP 3: Calculate order total with 8% tax rate\nSTEP 4: Check inventory for each item (loop through items, return mock availability)\nSTEP 5: Branch based on inventory - if all items available, create shipment record; otherwise create backorder record\nSTEP 6: Send confirmation (mock email to customer_email)\nSTEP 7: Return final order summary with status", - "expected": "ai_evals/fixtures/frontend/flow/expected/test2.json" - }, - { - "id": "flow-test5-parallel-data-pipeline", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nSTEP 1: Fetch list of data sources from configuration (return mock array of 3 source objects with id and url)\nSTEP 2: For each data source in parallel:\n - Fetch raw data from the source (mock fetch returning sample records)\n - Transform/clean the data (filter out invalid entries)\n - Validate the transformed data (return validation score 0-100)\nSTEP 3: Aggregate all validated data into single dataset with combined records\nSTEP 4: Calculate overall data quality score (average of all validation scores)\nSTEP 5: Branch based on quality score:\n - If score >= 90: Store in primary database and return success\n - If score >= 70 and < 90: Store in secondary database with warning flag\n - If score < 70: Store in quarantine and send alert\nSTEP 6: Return processing report with statistics (total records, quality score, destination)", - "expected": "ai_evals/fixtures/frontend/flow/expected/test3.json" - }, - { - "id": "flow-test6-ai-agent-tools", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nCreate a customer support flow with an AI agent:\n\nSTEP 1: Receive customer query from input (customer_id string, query_text string)\nSTEP 2: Fetch customer profile and order history (mock data based on customer_id)\nSTEP 3: Use an AI agent to handle the customer query. The agent should have access to these tools:\n - lookup_order: Takes order_id, returns order details (mock data)\n - check_refund_eligibility: Takes order_id, returns eligibility status and reason\n - create_support_ticket: Takes description and priority (low/medium/high), returns ticket_id\n - search_faq: Takes search_query, returns relevant FAQ answers\n The agent should use the customer profile context and respond helpfully.\nSTEP 4: Log the interaction to audit trail (customer_id, query, response summary)\nSTEP 5: Return the agent's response and any actions taken", - "expected": "ai_evals/fixtures/frontend/flow/expected/test4.json" - }, - { - "id": "flow-test7-simple-modification", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nModify this existing flow to add error handling:\n- Add a new step after process_data called \"validate_data\" to validate the processed data\n- The validation step should check if the data array is not empty\n- If validation fails (empty array), it should return an error object with message \"No data to save\"\n- If validation passes, return the data for the next step\n- Update save_results to handle the validation result appropriately", - "initial": "ai_evals/fixtures/frontend/flow/initial/test5_initial.json", - "expected": "ai_evals/fixtures/frontend/flow/expected/test5_modify_simple.json" - }, - { - "id": "flow-test8-branching-in-loop", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nModify the order processing loop to handle different order types:\n- Inside the loop_orders, replace the simple process_order step with branching based on order.type\n- For type \"express\": add a step called handle_express that marks as priority and calculates express shipping cost ($15.99)\n- For type \"standard\": add a step called handle_standard that calculates standard shipping cost ($5.99)\n- For type \"pickup\": add a step called handle_pickup that marks as no shipping required (cost $0)\n- Move the original process_order step to the default branch for unknown order types\n- Each branch step should return the orderId, shipping cost, and shipping type", - "initial": "ai_evals/fixtures/frontend/flow/initial/test6_initial.json", - "expected": "ai_evals/fixtures/frontend/flow/expected/test6_modify_medium.json" - }, - { - "id": "flow-test9-parallel-refactor", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nRefactor this flow for better performance by parallelizing the enrichment steps:\n- The three enrichment steps (enrich_price, enrich_inventory, enrich_reviews) currently run sequentially\n- Wrap them in a parallel branch (branchall) called \"parallel_enrichment\" so they run concurrently\n- Each enrichment step should include basic error handling with try/catch that returns a fallback value if it fails\n- Update the combine_data step to receive results from the parallel branch (results.parallel_enrichment returns an array of branch results)\n- The combine_data step should check if any enrichment used a fallback value and set a hasFallbacks flag\n- Keep get_item as the first step and return_result as the last step unchanged", - "initial": "ai_evals/fixtures/frontend/flow/initial/test7_initial.json", - "expected": "ai_evals/fixtures/frontend/flow/expected/test7_modify_complex.json" - }, - { - "id": "flow-test10-while-loop-counter", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nCreate a flow that counts upward until a target number is reached.\nThe flow input must contain a number field named target.\nUse a while loop step named \"count_until_target\".\nInside the loop, increment a current counter and keep looping until current >= target.\nAfter the loop, return the final counter value.\nKeep the flow minimal.", - "expected": "ai_evals/fixtures/frontend/flow/expected/test10_while_loop_counter.json" - }, - { - "id": "flow-test11-preprocessor-and-failure-handler", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nCreate an event processing flow with both a preprocessor module and a failure handler.\nThe flow input must include a payload string.\nUse a preprocessor module to trim the payload and reject empty strings.\nAdd one main rawscript step named \"process_event\" that returns a simple success object.\nAdd a failure module that returns a compact error object with message and step_id.\nKeep the flow minimal.", - "expected": "ai_evals/fixtures/frontend/flow/expected/test11_preprocessor_failure.json" - }, - { - "id": "flow-test12-approval-step", - "prompt": "THIS IS A TEST. PRODUCE MINIMAL FUNCTIONING FLOW CODE.\n\nCreate a flow for purchase approval.\nThe flow input must include requester_email and amount.\nAdd a rawscript step named \"request_approval\" that waits for approval with a resume form asking for approver_comment.\nRequire a single approval event before continuing.\nAfter approval, add a final rawscript step named \"finalize_purchase\" that returns an approved status object.\nKeep the flow minimal.", - "expected": "ai_evals/fixtures/frontend/flow/expected/test12_approval_step.json" - } -] diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml new file mode 100644 index 0000000000..60fb9ff704 --- /dev/null +++ b/ai_evals/cases/flow.yaml @@ -0,0 +1,186 @@ +- id: flow-test0-sum-two-numbers + prompt: |- + Create a flow that takes two numbers, `a` and `b`, and returns their sum. + Keep it simple and use a single step named `sum_numbers`. + expected: ai_evals/fixtures/frontend/flow/expected/test0_sum_two_numbers.json + +- id: flow-test1-reuse-existing-script + prompt: |- + I need a flow that adds two numbers. + If there is already a script in the workspace that does that, reuse it instead of rewriting the logic. + The flow should take `a` and `b` as inputs and use a single step named `sum_numbers`. + initial: ai_evals/fixtures/frontend/flow/initial/test1_reuse_existing_script_initial.json + expected: ai_evals/fixtures/frontend/flow/expected/test1_reuse_existing_script.json + +- id: flow-test2-call-existing-subflow + prompt: |- + Create a parent flow that adds two numbers by reusing an existing flow in the workspace if one already exists. + The parent flow should take `a` and `b` as inputs and delegate the calculation instead of inlining it. + Use a single step named `call_add_numbers`. + initial: ai_evals/fixtures/frontend/flow/initial/test2_call_existing_subflow_initial.json + expected: ai_evals/fixtures/frontend/flow/expected/test2_call_existing_subflow.json + +- id: flow-test3-branchone-routing + prompt: |- + Create a flow that routes incoming support requests based on the customer's tier. + The input should contain a string field named `tier`. + Free, pro, and enterprise requests should go to different queues, and unknown tiers should fall back to a default queue. + Name the main routing step `route_by_tier`. + expected: ai_evals/fixtures/frontend/flow/expected/test3_branchone_routing.json + +- id: flow-test4-order-processing-loop + prompt: |- + Build an order-processing flow. + + The input should include an order with: + - an `items` array containing `name`, `price`, and `quantity` + - `customer_email` + - `shipping_address` + + The flow should: + - validate that every item has a positive price and quantity + - calculate the order total with 8% tax + - check inventory for each item using placeholder availability data + - create a shipment if everything is in stock, otherwise create a backorder + - send a confirmation using placeholder email logic + - return a final order summary with the status + validate: + schemaRequiredPaths: + - order + - order.items + - order.customer_email + - order.shipping_address + resolveResultsRefs: true + modules: + - name: validates order item price and quantity + inputRefsContainAny: + - order + codeContainsAll: + - price + - quantity + - name: calculates totals with 8 percent tax + inputRefsContainAny: + - order + codeContainsAll: + - "0.08" + - name: checks inventory for each item + inputRefsContainAny: + - itemDetails + - order + codeContainsAll: + - available + codeContainsAny: + - ".map(" + - "forEach(" + - "for (" + - name: creates shipment outcome + codeContainsAny: + - shipment_id + - SHIP- + - name: creates backorder outcome + codeContainsAny: + - backorder_id + - BO- + - name: sends confirmation to the customer + inputRefsContainAny: + - order + codeContainsAny: + - customer_email + - customerEmail + - name: returns final summary using totals and confirmation + inputRefsContainAll: + - calculate_total + - send_confirmation + +- id: flow-test5-parallel-data-pipeline + prompt: |- + Create a data-processing flow for three external data sources. + + It should: + - load a small placeholder configuration listing the three sources + - fetch placeholder records from each source + - clean and validate each source's records + - combine everything into one dataset + - compute an overall quality score + - store the result differently depending on the score: + - 90 or above goes to the primary database + - 70 to 89 goes to a secondary database with a warning + - below 70 goes to quarantine and triggers an alert + - return a processing report with total records, quality score, and destination + expected: ai_evals/fixtures/frontend/flow/expected/test3.json + +- id: flow-test6-ai-agent-tools + prompt: |- + Create a customer support flow. + + The input should include `customer_id` and `query_text`. + The flow should load the customer's profile and order history, then use an AI assistant to help with the request. + The assistant should be able to: + - look up orders + - check refund eligibility + - search FAQs + - open a support ticket when needed + + After that, log the interaction and return the assistant's response along with any actions it took. + expected: ai_evals/fixtures/frontend/flow/expected/test4.json + +- id: flow-test7-simple-modification + prompt: |- + Update this flow so it validates processed data before saving it. + + After `process_data`, add a `validate_data` step that checks the data array is not empty. + If the array is empty, it should return an error object with the message `No data to save`. + If validation passes, let the save continue normally. + Update `save_results` so it handles the validation result correctly. + initial: ai_evals/fixtures/frontend/flow/initial/test5_initial.json + expected: ai_evals/fixtures/frontend/flow/expected/test5_modify_simple.json + +- id: flow-test8-branching-in-loop + prompt: |- + Update the order-processing logic inside `loop_orders` so different order types are handled differently. + + For `express`, mark the order as priority and use a shipping cost of $15.99. + For `standard`, use a shipping cost of $5.99. + For `pickup`, mark it as no shipping required with a cost of $0. + Keep the existing processing as a fallback for unknown order types. + Each path should return the orderId, shipping cost, and shipping type. + initial: ai_evals/fixtures/frontend/flow/initial/test6_initial.json + expected: ai_evals/fixtures/frontend/flow/expected/test6_modify_medium.json + +- id: flow-test9-parallel-refactor + prompt: |- + Refactor this flow so the enrichment work no longer runs one step at a time. + + `enrich_price`, `enrich_inventory`, and `enrich_reviews` should run independently. + Each one should return a fallback value if it fails. + Update `combine_data` so it merges the enrichment results and sets a `hasFallbacks` flag when any fallback was used. + Keep `get_item` as the first step and `return_result` as the last step. + initial: ai_evals/fixtures/frontend/flow/initial/test7_initial.json + expected: ai_evals/fixtures/frontend/flow/expected/test7_modify_complex.json + +- id: flow-test10-while-loop-counter + prompt: |- + Create a flow that keeps incrementing a counter until it reaches a target value. + The input should include a number field named `target`. + Name the looping step `count_until_target`. + Once the target is reached, return the final counter value. + expected: ai_evals/fixtures/frontend/flow/expected/test10_while_loop_counter.json + +- id: flow-test11-preprocessor-and-failure-handler + prompt: |- + Create an event-processing flow for a string payload. + + Before the main processing runs, trim the payload and reject empty strings. + The main step should be named `process_event` and return a simple success object. + If anything fails, return a compact error object with the error message and the failing step id. + expected: ai_evals/fixtures/frontend/flow/expected/test11_preprocessor_failure.json + +- id: flow-test12-approval-step + prompt: |- + Create a purchase approval flow. + + The input should include `requester_email` and `amount`. + Add an approval step named `request_approval` that pauses the flow and asks the approver for a comment. + One approval should be enough to continue. + After approval, add a final step named `finalize_purchase` that returns an approved status object. + expected: ai_evals/fixtures/frontend/flow/expected/test12_approval_step.json diff --git a/ai_evals/cases/script.json b/ai_evals/cases/script.json deleted file mode 100644 index 5d1c8b2ace..0000000000 --- a/ai_evals/cases/script.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "id": "script-test1-greet-user", - "prompt": "THIS IS A TEST, CODE SHOULD BE MINIMAL FUNCTIONING CODE.\n\nUpdate the current bun script so it exports `main(name: string)` and returns the plain string `Hello, ${name}!`.\nDo not return an object or array.\nDo not add external dependencies.", - "initial": "ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json", - "expected": "ai_evals/fixtures/frontend/script/expected/test1_greet_user.json" - } -] diff --git a/ai_evals/cases/script.yaml b/ai_evals/cases/script.yaml new file mode 100644 index 0000000000..bb1175586d --- /dev/null +++ b/ai_evals/cases/script.yaml @@ -0,0 +1,7 @@ +- id: script-test1-greet-user + prompt: |- + Update the current Bun script so it exports `main(name: string)` and returns the plain string `Hello, ${name}!`. + Do not return an object or array. + Do not add external dependencies. + initial: ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json + expected: ai_evals/fixtures/frontend/script/expected/test1_greet_user.json diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index cf55612490..0f904a2059 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -2,11 +2,24 @@ import { Command, InvalidArgumentError } from "commander"; import { loadCases, loadSelectedCases } from "../core/cases"; -import { buildRunResult, formatRunSummary, writeRunResult } from "../core/results"; +import { + EVAL_MODELS, + formatRunModelLabel, + getCliEvalModel, + getEvalModelHelpText, + resolveEvalModel, +} from "../core/models"; +import { + buildRunResult, + formatRunSummary, + resolveRunOutputPath, + writeRunArtifacts, + writeRunResult, +} from "../core/results"; import { runSuite } from "../core/runSuite"; import { EVAL_MODES, type EvalMode } from "../core/types"; import { DEFAULT_JUDGE_MODEL } from "../core/judge"; -import { createCliModeRunner, getCliRunModelLabel } from "../modes/cli"; +import { createCliModeRunner } from "../modes/cli"; import { runFrontendBenchmarkAdapter } from "../adapters/frontend/runtime"; async function main() { @@ -20,15 +33,27 @@ async function main() { [ "", "Examples:", + " bun run cli -- models", " bun run cli -- cases", " bun run cli -- cases flow", " bun run cli -- run flow", + " bun run cli -- run flow --model 4o", " bun run cli -- run flow flow-test0-sum-two-numbers --verbose", " bun run cli -- run flow flow-test5-simple-modification --runs 3", " bun run cli -- run cli bun-hello-script", + "", + "Models:", + getEvalModelHelpText(), ].join("\n") ); + program + .command("models") + .description("List available model aliases") + .action(() => { + handleModels(); + }); + program .command("cases") .description("List available cases") @@ -44,6 +69,7 @@ async function main() { .argument("[caseIds...]", "specific case ids to run") .option("--runs ", "number of attempts per case", parsePositiveInteger, 1) .option("--output ", "write the result JSON to this path") + .option("--model ", `model alias (${EVAL_MODELS.map((entry) => entry.id).join(", ")})`) .option("--verbose", "stream assistant output during frontend runs") .action( async ( @@ -52,6 +78,7 @@ async function main() { options: { runs: number; output?: string; + model?: string; verbose?: boolean; } ) => { @@ -60,6 +87,7 @@ async function main() { caseIds, runs: options.runs, outputPath: options.output, + model: options.model, verbose: options.verbose ?? false, }); } @@ -81,47 +109,73 @@ async function handleCases(mode?: EvalMode) { } } +function handleModels() { + process.stdout.write("Available models\n"); + for (const model of EVAL_MODELS) { + const supports = [ + ...(model.frontend ? ["flow", "script", "app"] : []), + ...(model.cli ? ["cli"] : []), + ]; + const aliases = [model.id, ...model.aliases.filter((alias) => alias !== model.id)]; + process.stdout.write(`- ${model.id}: ${model.label}\n`); + process.stdout.write(` aliases: ${aliases.join(", ")}\n`); + process.stdout.write(` modes: ${supports.join(", ")}\n`); + } + process.stdout.write(`\nJudge model: ${DEFAULT_JUDGE_MODEL}\n`); +} + async function handleRun(input: { mode: EvalMode; caseIds: string[]; runs: number; outputPath?: string; + model?: string; verbose: boolean; }) { const selectedCases = await loadSelectedCases(input.mode, input.caseIds); + const model = resolveEvalModel(input.mode, input.model); + const runModel = formatRunModelLabel(input.mode, model); process.stderr.write(`Starting ${input.mode} benchmark...\n`); const result = input.mode === "cli" - ? await runCliBenchmark(selectedCases, input.runs) + ? await runCliBenchmark(selectedCases, input.runs, getCliEvalModel(model), runModel) : await runFrontendBenchmarkAdapter({ mode: input.mode, caseIds: input.caseIds, runs: input.runs, + model: model.id, verbose: input.verbose, }); - const resultPath = await writeRunResult(result, input.outputPath); + const resolvedOutputPath = resolveRunOutputPath(input.mode, input.outputPath); + const artifactsPath = await writeRunArtifacts(result, resolvedOutputPath); + const resultPath = await writeRunResult(result, resolvedOutputPath); process.stdout.write(`${formatRunSummary(result)}\n`); process.stdout.write(`Saved: ${resultPath}\n`); + if (artifactsPath) { + process.stdout.write(`Artifacts: ${artifactsPath}\n`); + } } async function runCliBenchmark( cases: Awaited>, - runs: number + runs: number, + model: ReturnType, + runModel: string ) { const caseResults = await runSuite({ - modeRunner: createCliModeRunner(), + modeRunner: createCliModeRunner(model), cases, runs, - runModel: getCliRunModelLabel(), + runModel, judgeModel: DEFAULT_JUDGE_MODEL, }); return buildRunResult({ mode: "cli", runs, - runModel: getCliRunModelLabel(), + runModel, judgeModel: DEFAULT_JUDGE_MODEL, caseResults, }); diff --git a/ai_evals/core/cases.ts b/ai_evals/core/cases.ts index 440ab73a87..877ca44ed3 100644 --- a/ai_evals/core/cases.ts +++ b/ai_evals/core/cases.ts @@ -1,7 +1,8 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import type { EvalCase, EvalMode } from "./types"; +import { parse } from "yaml"; +import type { EvalCase, EvalMode, FlowValidationSpec } from "./types"; const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url)); const CASES_DIR = path.join(REPO_ROOT, "ai_evals", "cases"); @@ -11,6 +12,7 @@ interface RawEvalCase { prompt: string; initial?: string; expected?: string; + validate?: FlowValidationSpec; } export function getRepoRoot(): string { @@ -22,15 +24,16 @@ export function getAiEvalsRoot(): string { } export async function loadCases(mode: EvalMode): Promise { - const filePath = path.join(CASES_DIR, `${mode}.json`); + const filePath = path.join(CASES_DIR, `${mode}.yaml`); const raw = await readFile(filePath, "utf8"); - const parsed = JSON.parse(raw) as RawEvalCase[]; + const parsed = parse(raw) as RawEvalCase[]; return parsed.map((entry) => ({ id: entry.id, prompt: entry.prompt, initialPath: resolveFixturePath(entry.initial), expectedPath: resolveFixturePath(entry.expected), + validate: entry.validate, })); } diff --git a/ai_evals/core/judge.ts b/ai_evals/core/judge.ts index 6db81a4e8d..6308ddee82 100644 --- a/ai_evals/core/judge.ts +++ b/ai_evals/core/judge.ts @@ -29,8 +29,8 @@ export async function judgeOutput(input: { const system = [ "You evaluate benchmark outputs for Windmill AI generation.", "Deterministic checks already run separately. Focus on whether the final output satisfies the user request.", - "If expected state is provided, treat it as a strong reference and reward semantically equivalent outputs.", - "Be strict about missing requested functionality.", + "If expected state is provided, treat it as a valid example and reward semantically equivalent outputs.", + "Be strict about missing requested functionality. Inputs, module ids, and other details do not need to be exactly the same, but the functionality must be the same.", `Always respond by calling the ${JUDGE_TOOL_NAME} tool exactly once.`, ].join("\n\n"); diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts new file mode 100644 index 0000000000..db59df54aa --- /dev/null +++ b/ai_evals/core/models.ts @@ -0,0 +1,149 @@ +import type { EvalMode } from "./types"; + +export interface FrontendEvalModelConfig { + provider: "anthropic" | "openai"; + model: string; +} + +export interface CliEvalModelConfig { + provider: "anthropic"; + model: string; +} + +export interface EvalModelSpec { + id: string; + label: string; + aliases: string[]; + frontend?: FrontendEvalModelConfig; + cli?: CliEvalModelConfig; +} + +export const EVAL_MODELS: EvalModelSpec[] = [ + { + id: "haiku", + label: "Claude Haiku 4.5", + aliases: [ + "haiku", + "haiku-4.5", + "claude-haiku", + "claude-haiku-4.5", + "claude-haiku-4-5", + "claude-haiku-4-5-20251001", + ], + frontend: { + provider: "anthropic", + model: "claude-haiku-4-5-20251001", + }, + cli: { + provider: "anthropic", + model: "haiku", + }, + }, + { + id: "sonnet", + label: "Claude Sonnet 4.5", + aliases: [ + "sonnet", + "sonnet-4.5", + "claude-sonnet", + "claude-sonnet-4.5", + "claude-sonnet-4-5", + "claude-sonnet-4-5-20250929", + ], + frontend: { + provider: "anthropic", + model: "claude-sonnet-4-5-20250929", + }, + cli: { + provider: "anthropic", + model: "sonnet", + }, + }, + { + id: "opus", + label: "Claude Opus 4.6", + aliases: [ + "opus", + "opus-4.6", + "claude-opus", + "claude-opus-4.6", + "claude-opus-4-6", + ], + frontend: { + provider: "anthropic", + model: "claude-opus-4-6", + }, + cli: { + provider: "anthropic", + model: "opus", + }, + }, + { + id: "4o", + label: "GPT-4o", + aliases: ["4o", "gpt-4o"], + frontend: { + provider: "openai", + model: "gpt-4o", + }, + }, +]; + +export function resolveEvalModel(mode: EvalMode, alias?: string): EvalModelSpec { + const spec = alias ? findEvalModel(alias) : getDefaultEvalModel(mode); + if (!spec) { + throw new Error(`Unknown model: ${alias}`); + } + + if (mode === "cli" && !spec.cli) { + throw new Error(`Model ${spec.id} is not supported for cli mode`); + } + + if (mode !== "cli" && !spec.frontend) { + throw new Error(`Model ${spec.id} is not supported for ${mode} mode`); + } + + return spec; +} + +export function getEvalModelHelpText(): string { + return EVAL_MODELS.map((model) => { + const modes = [ + ...(model.frontend ? ["flow", "script", "app"] : []), + ...(model.cli ? ["cli"] : []), + ]; + return ` ${model.id.padEnd(8)} ${model.label} (${modes.join(", ")})`; + }).join("\n"); +} + +export function formatRunModelLabel(mode: EvalMode, model: EvalModelSpec): string { + if (mode === "cli") { + return `${model.cli!.provider}:${model.cli!.model}`; + } + return `${model.frontend!.provider}:${model.frontend!.model}`; +} + +export function getFrontendEvalModel(model: EvalModelSpec): FrontendEvalModelConfig { + if (!model.frontend) { + throw new Error(`Model ${model.id} does not support frontend evals`); + } + return model.frontend; +} + +export function getCliEvalModel(model: EvalModelSpec): CliEvalModelConfig { + if (!model.cli) { + throw new Error(`Model ${model.id} does not support cli evals`); + } + return model.cli; +} + +function getDefaultEvalModel(mode: EvalMode): EvalModelSpec { + return mode === "cli" ? EVAL_MODELS[0]! : EVAL_MODELS[0]!; +} + +function findEvalModel(alias: string): EvalModelSpec | undefined { + const normalized = alias.trim().toLowerCase(); + return EVAL_MODELS.find((model) => + [model.id, ...model.aliases].some((candidate) => candidate.toLowerCase() === normalized) + ); +} diff --git a/ai_evals/core/results.ts b/ai_evals/core/results.ts index 84dfb8b4cc..601e66ca93 100644 --- a/ai_evals/core/results.ts +++ b/ai_evals/core/results.ts @@ -1,8 +1,9 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { execFileSync } from "node:child_process"; import { getAiEvalsRoot, getRepoRoot } from "./cases"; import type { + BenchmarkArtifactFile, BenchmarkCaseResult, BenchmarkRunResult, EvalMode, @@ -12,13 +13,41 @@ export async function writeRunResult( result: BenchmarkRunResult, outputPath?: string ): Promise { - const targetPath = - outputPath ?? path.join(getAiEvalsRoot(), "results", defaultFileName(result.mode)); + const targetPath = resolveRunOutputPath(result.mode, outputPath); await mkdir(path.dirname(targetPath), { recursive: true }); - await writeFile(targetPath, JSON.stringify(result, null, 2) + "\n", "utf8"); + await writeFile(targetPath, JSON.stringify(toSerializableRunResult(result), null, 2) + "\n", "utf8"); return targetPath; } +export async function writeRunArtifacts( + result: BenchmarkRunResult, + outputPath?: string +): Promise { + const targetPath = resolveRunOutputPath(result.mode, outputPath); + const artifactRoot = defaultArtifactsRoot(targetPath); + + await rm(artifactRoot, { recursive: true, force: true }); + + let wroteArtifacts = false; + for (const caseResult of result.cases) { + for (const attempt of caseResult.attempts) { + const artifactFiles = attempt.artifactFiles ?? []; + if (artifactFiles.length === 0) { + attempt.artifactsPath = null; + continue; + } + + const attemptDir = path.join(artifactRoot, caseResult.id, `attempt-${attempt.attempt}`); + await writeArtifactFiles(attemptDir, artifactFiles); + attempt.artifactsPath = attemptDir; + wroteArtifacts = true; + } + } + + result.artifactsPath = wroteArtifacts ? artifactRoot : null; + return result.artifactsPath ?? null; +} + export function buildRunResult(input: { mode: EvalMode; runs: number; @@ -93,6 +122,47 @@ function defaultFileName(mode: EvalMode): string { return `${new Date().toISOString().replaceAll(":", "-")}__${mode}.json`; } +export function resolveRunOutputPath(mode: EvalMode, outputPath?: string): string { + return outputPath ?? path.join(getAiEvalsRoot(), "results", defaultFileName(mode)); +} + +function defaultArtifactsRoot(resultPath: string): string { + return resultPath.endsWith(".json") + ? resultPath.slice(0, -".json".length) + : `${resultPath}.artifacts`; +} + +async function writeArtifactFiles( + rootDir: string, + files: BenchmarkArtifactFile[] +): Promise { + for (const file of files) { + const relativePath = normalizeArtifactPath(file.path); + const targetPath = path.join(rootDir, relativePath); + await mkdir(path.dirname(targetPath), { recursive: true }); + await writeFile(targetPath, file.content, "utf8"); + } +} + +function normalizeArtifactPath(filePath: string): string { + const normalized = filePath.replaceAll("\\", "/").replace(/^\/+/, ""); + const parts = normalized.split("/").filter(Boolean); + if (parts.length === 0 || parts.some((part) => part === "." || part === "..")) { + throw new Error(`Invalid artifact path: ${filePath}`); + } + return parts.join("/"); +} + +function toSerializableRunResult(result: BenchmarkRunResult): BenchmarkRunResult { + return { + ...result, + cases: result.cases.map((caseResult) => ({ + ...caseResult, + attempts: caseResult.attempts.map(({ artifactFiles, ...attempt }) => attempt), + })), + }; +} + function getGitSha(): string | null { try { return execFileSync("git", ["rev-parse", "HEAD"], { diff --git a/ai_evals/core/runSuite.ts b/ai_evals/core/runSuite.ts index 28462c1ac8..19a3f62324 100644 --- a/ai_evals/core/runSuite.ts +++ b/ai_evals/core/runSuite.ts @@ -144,6 +144,7 @@ async function runCaseAttempts(input: { const checks: BenchmarkCheck[] = [ buildCheck("run succeeded", run.success, run.error), ...input.modeRunner.validate({ + evalCase: input.evalCase, prompt: input.evalCase.prompt, initial, expected, @@ -177,6 +178,7 @@ async function runCaseAttempts(input: { ); } + const artifactFiles = input.modeRunner.buildArtifacts?.(run.actual) ?? []; const attemptResult: BenchmarkAttemptResult = { attempt, passed: checks.every((check) => check.passed), @@ -189,6 +191,8 @@ async function runCaseAttempts(input: { judgeScore, judgeSummary, error: run.error ?? null, + artifactsPath: null, + artifactFiles, }; input.onProgress?.({ diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 5c7f23975d..6ba4713650 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -2,11 +2,30 @@ export const EVAL_MODES = ["cli", "flow", "script", "app"] as const; export type EvalMode = (typeof EVAL_MODES)[number]; +export interface FlowModuleValidation { + name: string; + typeAnyOf?: string[]; + inputRefsAny?: string[]; + inputRefsAll?: string[]; + inputRefsContainAny?: string[]; + inputRefsContainAll?: string[]; + codeContainsAny?: string[]; + codeContainsAll?: string[]; + codeRegexAll?: string[]; +} + +export interface FlowValidationSpec { + schemaRequiredPaths?: string[]; + resolveResultsRefs?: boolean; + modules?: FlowModuleValidation[]; +} + export interface EvalCase { id: string; prompt: string; initialPath?: string; expectedPath?: string; + validate?: FlowValidationSpec; } export interface BenchmarkCheck { @@ -22,6 +41,11 @@ export interface JudgeResult { error?: string; } +export interface BenchmarkArtifactFile { + path: string; + content: string; +} + export interface ModeRunOutput { success: boolean; actual: TActual; @@ -56,12 +80,14 @@ export interface ModeRunner { context: ModeRunContext ): Promise>; validate(input: { + evalCase: EvalCase; prompt: string; initial: TInitial | undefined; expected: TExpected | undefined; actual: TActual; run: ModeRunOutput; }): BenchmarkCheck[]; + buildArtifacts?(actual: TActual): BenchmarkArtifactFile[]; } export interface BenchmarkAttemptResult { @@ -76,6 +102,8 @@ export interface BenchmarkAttemptResult { judgeScore: number | null; judgeSummary: string | null; error: string | null; + artifactsPath?: string | null; + artifactFiles?: BenchmarkArtifactFile[]; } export interface BenchmarkCaseResult { @@ -99,6 +127,7 @@ export interface BenchmarkRunResult { passedAttempts: number; passRate: number; averageDurationMs: number; + artifactsPath?: string | null; cases: BenchmarkCaseResult[]; } diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 3353c9acbf..5b2a947ad5 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -1,5 +1,5 @@ import ts from "typescript"; -import type { BenchmarkCheck } from "./types"; +import type { BenchmarkCheck, FlowModuleValidation, FlowValidationSpec } from "./types"; export interface ScriptState { path: string; @@ -31,6 +31,7 @@ export interface AppRunnableState { } const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]); +const CONTROL_FLOW_MODULE_TYPES = new Set(["branchone", "branchall", "forloopflow", "whileloopflow"]); export function validateScriptState(input: { actual: ScriptState; @@ -81,6 +82,7 @@ export function validateFlowState(input: { actual: FlowState; initial?: FlowState; expected?: FlowState; + validate?: FlowValidationSpec; }): BenchmarkCheck[] { const actualModules = getFlowModules(input.actual); const checks: BenchmarkCheck[] = [check("flow has modules", actualModules.length > 0)]; @@ -95,6 +97,9 @@ export function validateFlowState(input: { } if (!input.expected) { + if (input.validate) { + checks.push(...validateFlowRequirements(input.actual, input.validate)); + } return checks; } @@ -231,6 +236,10 @@ export function validateFlowState(input: { } } + if (input.validate) { + checks.push(...validateFlowRequirements(input.actual, input.validate)); + } + return checks; } @@ -318,7 +327,7 @@ export function validateCliWorkspace(input: { } function check(name: string, passed: boolean, details?: string): BenchmarkCheck { - return details ? { name, passed, details } : { name, passed }; + return !passed && details ? { name, passed, details } : { name, passed }; } function normalizeText(value: string): string { @@ -341,8 +350,16 @@ function getScriptSyntaxErrors(code: string, lang: string): string[] { return []; } - const sourceFile = ts.createSourceFile("eval.ts", code, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS); - return sourceFile.parseDiagnostics.map((diagnostic) => + const result = ts.transpileModule(code, { + compilerOptions: { + target: ts.ScriptTarget.ES2022, + module: ts.ModuleKind.ESNext, + }, + reportDiagnostics: true, + fileName: "eval.ts", + }); + + return (result.diagnostics ?? []).map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n") ); } @@ -351,12 +368,452 @@ function getFlowModules(flow: FlowState): Array> { return Array.isArray(flow.value?.modules) ? flow.value.modules : []; } +function validateFlowRequirements( + flow: FlowState, + validate: FlowValidationSpec +): BenchmarkCheck[] { + const checks: BenchmarkCheck[] = []; + + for (const requiredPath of validate.schemaRequiredPaths ?? []) { + checks.push( + check( + `schema includes ${requiredPath}`, + hasSchemaPath(flow.schema, requiredPath), + `missing schema path ${requiredPath}` + ) + ); + } + + if (validate.resolveResultsRefs) { + const unresolved = collectUnresolvedResultsRefs(flow); + checks.push( + check( + "results references resolve", + unresolved.length === 0, + unresolved.length > 0 ? unresolved.join("; ") : undefined + ) + ); + } + + for (const requirement of validate.modules ?? []) { + const matched = findMatchingFlowModule(flow, requirement); + checks.push( + check( + requirement.name, + Boolean(matched), + matched ? undefined : describeMissingModuleRequirement(requirement) + ) + ); + } + + return checks; +} + function getTopLevelFlowModuleIds(flow: FlowState): string[] { return getFlowModules(flow) .map((module) => module.id) .filter((value): value is string => typeof value === "string"); } +function hasSchemaPath(schema: Record | undefined, dottedPath: string): boolean { + if (!schema || typeof schema !== "object") { + return false; + } + + const segments = dottedPath.split(".").filter(Boolean); + if (segments.length === 0) { + return false; + } + + let current: Record | undefined = schema; + for (const segment of segments) { + const properties = current?.properties; + if (!properties || typeof properties !== "object") { + return false; + } + + const next = (properties as Record)[segment]; + if (!next || typeof next !== "object") { + return false; + } + current = next as Record; + } + + return true; +} + +function collectUnresolvedResultsRefs(flow: FlowState): string[] { + const unresolved = new Set(); + validateModuleSequence(getFlowModules(flow), new Map>(), unresolved); + return [...unresolved]; +} + +function validateModuleSequence( + modules: Array>, + parentVisibleModules: Map>, + unresolved: Set +): void { + const visibleModules = new Map(parentVisibleModules); + + for (const module of modules) { + validateResultsRefsInRecord(module, visibleModules, unresolved); + validateNestedModuleResultsRefs(module, visibleModules, unresolved); + + if (typeof module.id === "string" && module.id.length > 0) { + visibleModules.set(module.id, module); + } + } +} + +function validateNestedModuleResultsRefs( + module: Record, + visibleModules: Map>, + unresolved: Set +): void { + const value = isObjectRecord(module.value) ? module.value : null; + if (!value) { + return; + } + + const nestedSequences: Array>> = []; + + if (Array.isArray(value.modules)) { + nestedSequences.push(asModuleArray(value.modules)); + } + + if (Array.isArray(value.default)) { + nestedSequences.push(asModuleArray(value.default)); + } + + if (Array.isArray(value.branches)) { + for (const branch of value.branches) { + if (!isObjectRecord(branch)) { + continue; + } + if (typeof branch.expr === "string") { + validateResultsRefsInExpression( + branch.expr, + `branch ${module.id ?? "(unnamed)"}`, + visibleModules, + unresolved + ); + } + if (Array.isArray(branch.modules)) { + nestedSequences.push(asModuleArray(branch.modules)); + } + } + } + + for (const sequence of nestedSequences) { + validateModuleSequence(sequence, visibleModules, unresolved); + } +} + +function validateResultsRefsInRecord( + value: unknown, + visibleModules: Map>, + unresolved: Set, + context = "expression" +): void { + if (typeof value === "string") { + validateResultsRefsInExpression(value, context, visibleModules, unresolved); + return; + } + + if (Array.isArray(value)) { + for (const entry of value) { + validateResultsRefsInRecord(entry, visibleModules, unresolved, context); + } + return; + } + + if (!isObjectRecord(value)) { + return; + } + + for (const [key, entry] of Object.entries(value)) { + if (key === "content" || key === "modules" || key === "branches" || key === "default") { + continue; + } + validateResultsRefsInRecord(entry, visibleModules, unresolved, key); + } +} + +function validateResultsRefsInExpression( + expression: string, + context: string, + visibleModules: Map>, + unresolved: Set +): void { + for (const ref of extractResultsRefs(expression)) { + const module = visibleModules.get(ref.root); + if (!module) { + unresolved.add(`${context} references missing results.${ref.root}`); + continue; + } + validateNestedResultsRefPath(ref.root, ref.path, module, context, unresolved); + } +} + +function extractResultsRefs( + expression: string +): Array<{ root: string; path: string[] }> { + const matches = expression.matchAll(/\bresults\.([A-Za-z0-9_-]+)((?:\.[A-Za-z0-9_-]+)*)/g); + const refs = new Map(); + + for (const match of matches) { + const root = match[1]; + const path = match[2] + .split(".") + .filter(Boolean); + const key = `${root}:${path.join(".")}`; + refs.set(key, { root, path }); + } + + return [...refs.values()]; +} + +function validateNestedResultsRefPath( + rootId: string, + path: string[], + module: Record, + context: string, + unresolved: Set +): void { + if (path.length === 0) { + return; + } + + const moduleType = getModuleType(module); + if (!moduleType || !CONTROL_FLOW_MODULE_TYPES.has(moduleType)) { + return; + } + + const nestedIds = new Set(getImmediateNestedModuleIds(module)); + const [firstSegment] = path; + if (nestedIds.has(firstSegment)) { + unresolved.add( + `${context} references nested results.${rootId}.${firstSegment} inside ${moduleType} ${rootId}` + ); + } +} + +function findMatchingFlowModule( + flow: FlowState, + requirement: FlowModuleValidation +): Record | null { + for (const module of getAllFlowModules(flow)) { + if (matchesFlowModuleRequirement(module, requirement)) { + return module; + } + } + return null; +} + +function matchesFlowModuleRequirement( + module: Record, + requirement: FlowModuleValidation +): boolean { + if (requirement.typeAnyOf && requirement.typeAnyOf.length > 0) { + const actualType = getModuleType(module); + if (!actualType || !requirement.typeAnyOf.includes(actualType)) { + return false; + } + } + + const inputRefs = getModuleInputRefs(module); + if (requirement.inputRefsAny && requirement.inputRefsAny.length > 0) { + if (!requirement.inputRefsAny.some((ref) => inputRefs.includes(ref))) { + return false; + } + } + + if (requirement.inputRefsAll && requirement.inputRefsAll.length > 0) { + if (!requirement.inputRefsAll.every((ref) => inputRefs.includes(ref))) { + return false; + } + } + + if (requirement.inputRefsContainAny && requirement.inputRefsContainAny.length > 0) { + if (!requirement.inputRefsContainAny.some((snippet) => inputRefs.some((ref) => ref.includes(snippet)))) { + return false; + } + } + + if (requirement.inputRefsContainAll && requirement.inputRefsContainAll.length > 0) { + if (!requirement.inputRefsContainAll.every((snippet) => inputRefs.some((ref) => ref.includes(snippet)))) { + return false; + } + } + + const code = getModuleCode(module); + if (requirement.codeContainsAny && requirement.codeContainsAny.length > 0) { + if (!code || !requirement.codeContainsAny.some((snippet) => code.includes(snippet))) { + return false; + } + } + + if (requirement.codeContainsAll && requirement.codeContainsAll.length > 0) { + if (!code || !requirement.codeContainsAll.every((snippet) => code.includes(snippet))) { + return false; + } + } + + if (requirement.codeRegexAll && requirement.codeRegexAll.length > 0) { + if (!code) { + return false; + } + for (const pattern of requirement.codeRegexAll) { + const regex = new RegExp(pattern, "m"); + if (!regex.test(code)) { + return false; + } + } + } + + return true; +} + +function describeMissingModuleRequirement(requirement: FlowModuleValidation): string { + const parts: string[] = []; + if (requirement.typeAnyOf?.length) { + parts.push(`type in [${requirement.typeAnyOf.join(", ")}]`); + } + if (requirement.inputRefsAny?.length) { + parts.push(`any input refs [${requirement.inputRefsAny.join(", ")}]`); + } + if (requirement.inputRefsAll?.length) { + parts.push(`all input refs [${requirement.inputRefsAll.join(", ")}]`); + } + if (requirement.inputRefsContainAny?.length) { + parts.push(`any input refs containing [${requirement.inputRefsContainAny.join(", ")}]`); + } + if (requirement.inputRefsContainAll?.length) { + parts.push(`all input refs containing [${requirement.inputRefsContainAll.join(", ")}]`); + } + if (requirement.codeContainsAny?.length) { + parts.push(`code contains any [${requirement.codeContainsAny.join(", ")}]`); + } + if (requirement.codeContainsAll?.length) { + parts.push(`code contains all [${requirement.codeContainsAll.join(", ")}]`); + } + if (requirement.codeRegexAll?.length) { + parts.push(`code matches [${requirement.codeRegexAll.join(", ")}]`); + } + return parts.join("; "); +} + +function getAllFlowModules(flow: FlowState): Array> { + const modules: Array> = []; + const specialModules = ["preprocessor_module", "failure_module"] as const; + + for (const key of specialModules) { + const specialModule = getSpecialFlowModule(flow, key); + if (specialModule) { + modules.push(specialModule); + modules.push(...collectNestedModules(specialModule)); + } + } + + for (const module of getFlowModules(flow)) { + modules.push(module); + modules.push(...collectNestedModules(module)); + } + + return modules; +} + +function collectNestedModules(module: Record): Array> { + const nested: Array> = []; + const value = isObjectRecord(module.value) ? module.value : null; + if (!value) { + return nested; + } + + if (Array.isArray(value.modules)) { + for (const child of asModuleArray(value.modules)) { + nested.push(child, ...collectNestedModules(child)); + } + } + + if (Array.isArray(value.default)) { + for (const child of asModuleArray(value.default)) { + nested.push(child, ...collectNestedModules(child)); + } + } + + if (Array.isArray(value.branches)) { + for (const branch of value.branches) { + if (!isObjectRecord(branch) || !Array.isArray(branch.modules)) { + continue; + } + for (const child of asModuleArray(branch.modules)) { + nested.push(child, ...collectNestedModules(child)); + } + } + } + + return nested; +} + +function getImmediateNestedModuleIds(module: Record): string[] { + const ids: string[] = []; + const value = isObjectRecord(module.value) ? module.value : null; + if (!value) { + return ids; + } + + if (Array.isArray(value.modules)) { + ids.push(...asModuleArray(value.modules).flatMap((child) => (typeof child.id === "string" ? [child.id] : []))); + } + + if (Array.isArray(value.default)) { + ids.push(...asModuleArray(value.default).flatMap((child) => (typeof child.id === "string" ? [child.id] : []))); + } + + if (Array.isArray(value.branches)) { + for (const branch of value.branches) { + if (!isObjectRecord(branch) || !Array.isArray(branch.modules)) { + continue; + } + ids.push( + ...asModuleArray(branch.modules).flatMap((child) => (typeof child.id === "string" ? [child.id] : [])) + ); + } + } + + return ids; +} + +function getModuleInputRefs(module: Record): string[] { + const value = isObjectRecord(module.value) ? module.value : null; + const inputTransforms = isObjectRecord(value?.input_transforms) ? value?.input_transforms : null; + if (!inputTransforms) { + return []; + } + + return Object.values(inputTransforms) + .flatMap((entry) => { + if (!isObjectRecord(entry) || typeof entry.expr !== "string") { + return []; + } + return [entry.expr.trim()]; + }); +} + +function getModuleCode(module: Record): string | null { + const value = isObjectRecord(module.value) ? module.value : null; + return typeof value?.content === "string" ? value.content : null; +} + +function asModuleArray(value: unknown[]): Array> { + return value.filter(isObjectRecord); +} + +function isObjectRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + function getSpecialFlowModule( flow: FlowState, key: "preprocessor_module" | "failure_module" diff --git a/ai_evals/modes/app.ts b/ai_evals/modes/app.ts index bfcd74afa7..dd332aa2f0 100644 --- a/ai_evals/modes/app.ts +++ b/ai_evals/modes/app.ts @@ -1,11 +1,14 @@ import { loadAppFixture } from "../adapters/frontend/core/app/appFixtureLoader"; import type { AppFiles } from "../../frontend/src/lib/components/copilot/chat/app/core"; +import type { FrontendEvalModelConfig } from "../core/models"; import { validateAppState, type AppFilesState } from "../core/validators"; -import type { ModeRunner } from "../core/types"; +import type { BenchmarkArtifactFile, ModeRunner } from "../core/types"; import { runAppEval } from "../adapters/frontend/core/app/appEvalRunner"; -import { FRONTEND_MODEL, FRONTEND_PROVIDER, getFrontendApiKey } from "./frontendCommon"; +import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon"; -export function createAppModeRunner(): ModeRunner { +export function createAppModeRunner( + modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL +): ModeRunner { return { mode: "app", concurrency: 5, @@ -17,11 +20,11 @@ export function createAppModeRunner(): ModeRunner { +export function createCliModeRunner( + modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL +): ModeRunner { return { mode: "cli", concurrency: 1, @@ -66,7 +74,7 @@ export function createCliModeRunner(): ModeRunner { diff --git a/ai_evals/modes/flow.ts b/ai_evals/modes/flow.ts index 14bf62e467..4e74da4f42 100644 --- a/ai_evals/modes/flow.ts +++ b/ai_evals/modes/flow.ts @@ -1,19 +1,22 @@ import { readJsonFile } from "../core/files"; +import type { FrontendEvalModelConfig } from "../core/models"; import { validateFlowState, type FlowState } from "../core/validators"; -import type { ModeRunner } from "../core/types"; +import type { BenchmarkArtifactFile, ModeRunner } from "../core/types"; import { runFlowEval, type FlowFixture, } from "../adapters/frontend/core/flow/flowEvalRunner"; import type { FlowWorkspaceFixtures } from "../adapters/frontend/core/flow/fileHelpers"; -import { FRONTEND_MODEL, FRONTEND_PROVIDER, getFrontendApiKey } from "./frontendCommon"; +import { DEFAULT_FRONTEND_EVAL_MODEL, getFrontendApiKey } from "./frontendCommon"; interface FlowInitialFixture { flow?: FlowFixture; workspace?: FlowWorkspaceFixtures; } -export function createFlowModeRunner(): ModeRunner { +export function createFlowModeRunner( + modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL +): ModeRunner { return { mode: "flow", concurrency: 5, @@ -31,11 +34,11 @@ export function createFlowModeRunner(): ModeRunner(path)); }, async run(prompt, initial, context) { - const result = await runFlowEval(prompt, getFrontendApiKey(), { + const result = await runFlowEval(prompt, getFrontendApiKey(modelConfig.provider), { initialFlow: initial?.flow, workspaceFixtures: initial?.workspace, - provider: FRONTEND_PROVIDER, - model: FRONTEND_MODEL, + provider: modelConfig.provider, + model: modelConfig.model, runContext: context, }); @@ -52,8 +55,21 @@ export function createFlowModeRunner(): ModeRunner { +export function createScriptModeRunner( + modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL +): ModeRunner { return { mode: "script", concurrency: 5, @@ -21,10 +24,10 @@ export function createScriptModeRunner(): ModeRunner