feat: add benchmark cli compare command

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

ai_evals/README.md

@@ -55,8 +55,10 @@ Current usage:
 
 ```bash
 cd ai_evals
+bun run cli -- list-variants --surface cli
 bun run cli -- list-cases --surface cli
-bun run cli -- run --surface cli --case bun-hello-script
+bun run cli -- run --surface cli --case bun-hello-script --variant baseline
+bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline
 ```
 
 At the moment this is still intentionally small, but it is the only benchmark

ai_evals/adapters/cli/artifact-eval.ts

@@ -4,11 +4,11 @@ import { tmpdir } from "os";
 import { dirname, join } from "path";
 import { fileURLToPath } from "url";
 import {
-  getGeneratedSkillsSource,
   runPromptAndCapture,
   type PromptRunResult,
   wasSkillInvoked
 } from "./runtime";
+import type { CliVariant } from "./variants";
 
 export interface ExpectedFile {
   path: string;
@@ -46,6 +46,7 @@ export interface CliArtifactEvalResult {
   checks: ArtifactCheck[];
   expectedFiles: FileArtifactResult[];
   passed: boolean;
+  variantId: string;
 }
 
 const CASES_DIR = fileURLToPath(new URL("../../cases/cli", import.meta.url));
@@ -72,9 +73,12 @@ export async function loadCliArtifactEvalCases(): Promise<CliArtifactEvalCase[]>
 }
 
 export async function runCliArtifactEvalCase(
-  evalCase: CliArtifactEvalCase
+  evalCase: CliArtifactEvalCase,
+  options: {
+    variant: CliVariant;
+  }
 ): Promise<CliArtifactEvalResult> {
-  const workspaceDir = await createIsolatedWorkspace(evalCase.id);
+  const workspaceDir = await createIsolatedWorkspace(evalCase.id, options.variant.skillsSourcePath);
 
   try {
     const renderedPrompt = renderPrompt(evalCase.prompt, workspaceDir);
@@ -92,7 +96,8 @@ export async function runCliArtifactEvalCase(
       run,
       checks,
       expectedFiles: fileResults,
-      passed: checks.every((check) => check.required === false || check.passed)
+      passed: checks.every((check) => check.required === false || check.passed),
+      variantId: options.variant.id
     };
   } catch (error) {
     if (!shouldKeepWorkspace()) {
@@ -110,13 +115,15 @@ export function shouldKeepWorkspace(): boolean {
   return process.env.WMILL_CLI_EVAL_KEEP_WORKSPACE === "1";
 }
 
-async function createIsolatedWorkspace(caseId: string): Promise<string> {
+async function createIsolatedWorkspace(
+  caseId: string,
+  skillsSourcePath: string
+): Promise<string> {
   const workspaceDir = await mkdtemp(join(tmpdir(), `wmill-cli-artifact-${caseId}-`));
   const skillsDir = join(workspaceDir, ".claude", "skills");
-  const generatedSkillsSource = getGeneratedSkillsSource();
 
   await mkdir(dirname(skillsDir), { recursive: true });
-  await cp(generatedSkillsSource, skillsDir, { recursive: true });
+  await cp(skillsSourcePath, skillsDir, { recursive: true });
   await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8");
 
   return workspaceDir;

ai_evals/adapters/cli/variants.ts

@@ -0,0 +1,73 @@
+import { readdir, readFile } from "fs/promises";
+import { join, resolve } from "path";
+import { fileURLToPath } from "url";
+import { getGeneratedSkillsSource } from "./runtime";
+
+type CliVariantSource =
+  | {
+      type: "generated";
+    }
+  | {
+      type: "path";
+      path: string;
+    };
+
+interface CliVariantManifest {
+  id: string;
+  description?: string;
+  skillsSource: CliVariantSource;
+}
+
+export interface CliVariant {
+  id: string;
+  description?: string;
+  skillsSourcePath: string;
+}
+
+const VARIANTS_DIR = fileURLToPath(new URL("../../variants/cli", import.meta.url));
+
+export async function loadCliVariants(): Promise<CliVariant[]> {
+  const filenames = (await readdir(VARIANTS_DIR))
+    .filter((entry) => entry.endsWith(".json"))
+    .sort((left, right) => left.localeCompare(right));
+
+  const variants: CliVariant[] = [];
+
+  for (const filename of filenames) {
+    const manifestPath = join(VARIANTS_DIR, filename);
+    const raw = await readFile(manifestPath, "utf8");
+    const parsed = JSON.parse(raw) as CliVariantManifest;
+
+    if (!parsed.id) {
+      throw new Error(`Missing variant id in ${manifestPath}`);
+    }
+
+    variants.push({
+      id: parsed.id,
+      description: parsed.description,
+      skillsSourcePath: resolveVariantSkillsSource(parsed.skillsSource, manifestPath)
+    });
+  }
+
+  return variants;
+}
+
+export async function loadCliVariantById(variantId: string): Promise<CliVariant> {
+  const variants = await loadCliVariants();
+  const variant = variants.find((entry) => entry.id === variantId);
+  if (!variant) {
+    throw new Error(`Unknown CLI variant: ${variantId}`);
+  }
+  return variant;
+}
+
+function resolveVariantSkillsSource(
+  skillsSource: CliVariantSource | undefined,
+  manifestPath: string
+): string {
+  if (!skillsSource || skillsSource.type === "generated") {
+    return getGeneratedSkillsSource();
+  }
+
+  return resolve(join(manifestPath, ".."), skillsSource.path);
+}

ai_evals/cli/README.md

@@ -28,25 +28,39 @@ cd ai_evals
 bun run cli -- list-cases --surface cli
 ```
 
+List available CLI variants:
+
+```bash
+cd ai_evals
+bun run cli -- list-variants --surface cli
+```
+
 Run one CLI case:
 
 ```bash
 cd ai_evals
-bun run cli -- run --surface cli --case bun-hello-script
+bun run cli -- run --surface cli --case bun-hello-script --variant baseline
 ```
 
 Keep the temp workspace for inspection:
 
 ```bash
 cd ai_evals
-bun run cli -- run --surface cli --case bun-hello-script --keep-workspace
+bun run cli -- run --surface cli --case bun-hello-script --variant baseline --keep-workspace
 ```
 
 Print machine-readable output:
 
 ```bash
 cd ai_evals
-bun run cli -- run --surface cli --case bun-hello-script --json
+bun run cli -- run --surface cli --case bun-hello-script --variant baseline --json
+```
+
+Compare two variant selections on one or more cases:
+
+```bash
+cd ai_evals
+bun run cli -- compare --surface cli --case bun-hello-script --variant baseline --variant baseline --json
 ```
 
 ## Next Steps

ai_evals/cli/index.ts

@@ -5,14 +5,16 @@ import {
   loadCliArtifactEvalCases,
   runCliArtifactEvalCase
 } from "../adapters/cli/artifact-eval";
+import { loadCliVariantById, loadCliVariants, type CliVariant } from "../adapters/cli/variants";
 
-type CommandName = "run" | "list-cases" | "compare" | "history";
+type CommandName = "run" | "list-cases" | "list-variants" | "compare" | "history";
 type SurfaceName = "cli";
 
 interface ParsedArgs {
   command: CommandName;
   surface?: string;
-  caseId?: string;
+  caseIds: string[];
+  variantIds: string[];
   json: boolean;
   keepWorkspace: boolean;
 }
@@ -24,10 +26,15 @@ async function main() {
     case "list-cases":
       await handleListCases(args);
       return;
+    case "list-variants":
+      await handleListVariants(args);
+      return;
     case "run":
       await handleRun(args);
       return;
     case "compare":
+      await handleCompare(args);
+      return;
     case "history":
       throw new Error(
         `'${args.command}' is not implemented yet in the repo-level benchmark CLI`
@@ -69,9 +76,41 @@ async function handleListCases(args: ParsedArgs) {
   }
 }
 
+async function handleListVariants(args: ParsedArgs) {
+  const surface = requireSurface(args.surface);
+
+  switch (surface) {
+    case "cli": {
+      const variants = await loadCliVariants();
+      const payload = {
+        surface,
+        variants: variants.map((entry) => ({
+          id: entry.id,
+          description: entry.description ?? null
+        }))
+      };
+
+      if (args.json) {
+        process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
+        return;
+      }
+
+      process.stdout.write(`Surface: ${surface}\n`);
+      for (const entry of payload.variants) {
+        process.stdout.write(
+          `- ${entry.id}${entry.description ? `: ${entry.description}` : ""}\n`
+        );
+      }
+      return;
+    }
+    default:
+      assertNever(surface);
+  }
+}
+
 async function handleRun(args: ParsedArgs) {
   const surface = requireSurface(args.surface);
-  const caseId = requireCaseId(args.caseId);
+  const caseId = requireSingleCaseId(args.caseIds);
 
   switch (surface) {
     case "cli": {
@@ -80,11 +119,13 @@ async function handleRun(args: ParsedArgs) {
       if (!evalCase) {
         throw new Error(`Unknown CLI case: ${caseId}`);
       }
+      const variant = await loadCliVariantById(requireSingleVariantId(args.variantIds, "baseline"));
 
-      const result = await runCliArtifactEvalCase(evalCase);
+      const result = await runCliArtifactEvalCase(evalCase, { variant });
       const payload = {
         command: "run",
         surface,
+        variant: variant.id,
         caseId: evalCase.id,
         passed: result.passed,
         workspaceKept: args.keepWorkspace,
@@ -120,6 +161,83 @@ async function handleRun(args: ParsedArgs) {
   }
 }
 
+async function handleCompare(args: ParsedArgs) {
+  const surface = requireSurface(args.surface);
+
+  switch (surface) {
+    case "cli": {
+      const allCases = await loadCliArtifactEvalCases();
+      const selectedCases =
+        args.caseIds.length === 0
+          ? allCases
+          : args.caseIds.map((caseId) => {
+              const evalCase = allCases.find((entry) => entry.id === caseId);
+              if (!evalCase) {
+                throw new Error(`Unknown CLI case: ${caseId}`);
+              }
+              return evalCase;
+            });
+
+      if (args.variantIds.length < 2) {
+        throw new Error("compare requires at least two --variant values");
+      }
+
+      const variants = await Promise.all(
+        args.variantIds.map((variantId) => loadCliVariantById(variantId))
+      );
+      const labeledVariants = labelVariantSelections(variants);
+      const variantResults = [];
+
+      for (const labeledVariant of labeledVariants) {
+        const caseResults = [];
+
+        for (const evalCase of selectedCases) {
+          const result = await runCliArtifactEvalCase(evalCase, {
+            variant: labeledVariant.variant
+          });
+
+          try {
+            caseResults.push({
+              caseId: evalCase.id,
+              passed: result.passed,
+              skillsInvoked: result.run.skillsInvoked,
+              toolsUsed: result.run.toolsUsed.map((tool) => tool.tool),
+              checks: result.checks
+            });
+          } finally {
+            await cleanupWorkspace(result.workspaceDir);
+          }
+        }
+
+        variantResults.push({
+          label: labeledVariant.label,
+          variant: labeledVariant.variant.id,
+          totalCases: caseResults.length,
+          passedCases: caseResults.filter((entry) => entry.passed).length,
+          caseResults
+        });
+      }
+
+      const payload = {
+        command: "compare",
+        surface,
+        caseIds: selectedCases.map((entry) => entry.id),
+        variants: variantResults
+      };
+
+      if (args.json) {
+        process.stdout.write(JSON.stringify(payload, null, 2) + "\n");
+      } else {
+        printCompareSummary(payload);
+      }
+
+      return;
+    }
+    default:
+      assertNever(surface);
+  }
+}
+
 function parseArgs(argv: string[]): ParsedArgs {
   const [commandArg, ...rest] = argv;
 
@@ -134,6 +252,8 @@ function parseArgs(argv: string[]): ParsedArgs {
 
   const parsed: ParsedArgs = {
     command: commandArg,
+    caseIds: [],
+    variantIds: [],
     json: false,
     keepWorkspace: false
   };
@@ -148,7 +268,13 @@ function parseArgs(argv: string[]): ParsedArgs {
     }
 
     if (arg === "--case") {
-      parsed.caseId = rest[index + 1];
+      parsed.caseIds.push(rest[index + 1]);
+      index += 1;
+      continue;
+    }
+
+    if (arg === "--variant") {
+      parsed.variantIds.push(rest[index + 1]);
       index += 1;
       continue;
     }
@@ -178,6 +304,7 @@ function isCommandName(value: string): value is CommandName {
   return (
     value === "run" ||
     value === "list-cases" ||
+    value === "list-variants" ||
     value === "compare" ||
     value === "history"
   );
@@ -193,15 +320,29 @@ function requireSurface(surface: string | undefined): SurfaceName {
   return surface;
 }
 
-function requireCaseId(caseId: string | undefined): string {
-  if (!caseId) {
+function requireSingleCaseId(caseIds: string[]): string {
+  if (caseIds.length === 0) {
     throw new Error("Missing required --case argument");
   }
-  return caseId;
+  if (caseIds.length > 1) {
+    throw new Error("run accepts only one --case value");
+  }
+  return caseIds[0];
+}
+
+function requireSingleVariantId(variantIds: string[], fallback: string): string {
+  if (variantIds.length === 0) {
+    return fallback;
+  }
+  if (variantIds.length > 1) {
+    throw new Error("run accepts only one --variant value");
+  }
+  return variantIds[0];
 }
 
 function printRunSummary(payload: {
   surface: SurfaceName;
+  variant: string;
   caseId: string;
   passed: boolean;
   workspaceKept: boolean;
@@ -212,6 +353,7 @@ function printRunSummary(payload: {
   expectedFiles: Array<{ path: string; exists: boolean }>;
 }) {
   process.stdout.write(`Surface: ${payload.surface}\n`);
+  process.stdout.write(`Variant: ${payload.variant}\n`);
   process.stdout.write(`Case: ${payload.caseId}\n`);
   process.stdout.write(`Passed: ${payload.passed ? "yes" : "no"}\n`);
   process.stdout.write(`Skills: ${payload.skillsInvoked.join(", ") || "(none)"}\n`);
@@ -233,13 +375,57 @@ function printRunSummary(payload: {
   }
 }
 
+function printCompareSummary(payload: {
+  surface: SurfaceName;
+  caseIds: string[];
+  variants: Array<{
+    label: string;
+    variant: string;
+    totalCases: number;
+    passedCases: number;
+    caseResults: Array<{ caseId: string; passed: boolean }>;
+  }>;
+}) {
+  process.stdout.write(`Surface: ${payload.surface}\n`);
+  process.stdout.write(`Cases: ${payload.caseIds.join(", ")}\n`);
+  process.stdout.write("Variants:\n");
+
+  for (const variant of payload.variants) {
+    process.stdout.write(
+      `- ${variant.label}: ${variant.passedCases}/${variant.totalCases} cases passed\n`
+    );
+    for (const caseResult of variant.caseResults) {
+      process.stdout.write(
+        `  ${caseResult.caseId}: ${caseResult.passed ? "pass" : "fail"}\n`
+      );
+    }
+  }
+}
+
+function labelVariantSelections(
+  variants: CliVariant[]
+): Array<{ label: string; variant: CliVariant }> {
+  const seenCounts = new Map<string, number>();
+
+  return variants.map((variant) => {
+    const count = (seenCounts.get(variant.id) ?? 0) + 1;
+    seenCounts.set(variant.id, count);
+
+    return {
+      label: count === 1 ? variant.id : `${variant.id}#${count}`,
+      variant
+    };
+  });
+}
+
 function printHelp() {
   process.stdout.write(
     [
       "Usage:",
       "  cd ai_evals && bun run cli -- list-cases --surface cli [--json]",
-      "  cd ai_evals && bun run cli -- run --surface cli --case <id> [--json] [--keep-workspace]",
-      "  cd ai_evals && bun run cli -- compare",
+      "  cd ai_evals && bun run cli -- list-variants --surface cli [--json]",
+      "  cd ai_evals && bun run cli -- run --surface cli --case <id> [--variant <id>] [--json] [--keep-workspace]",
+      "  cd ai_evals && bun run cli -- compare --surface cli [--case <id> ...] [--variant <id> ...] [--json]",
       "  cd ai_evals && bun run cli -- history",
       "",
       "Current support:",

ai_evals/variants/cli/baseline.json

@@ -0,0 +1,7 @@
+{
+  "id": "baseline",
+  "description": "Uses the repo's generated Windmill CLI skills.",
+  "skillsSource": {
+    "type": "generated"
+  }
+}